From c4870b27bcb4a6f70606ccef9adeded948c38f0c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 02:45:07 +0700 Subject: [PATCH 01/18] feat: canonical reference rows for indexed-tree secondaries (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the placeholder indexed-secondary row representation with canonical one-hop references back to the primary entry, before indexed trees ship. Every axis now stores the same element family: ReferenceWithSumItem(SiblingReference(primary_key), Some(1), axis_payload_sum) written as a COMBINED reference so the row's committed value hash is combine_hash(H(reference bytes), primary_node_value_hash). Key decisions (per the issue's converged review): - All three axes stay on the dual-aggregate ProvableCountProvableSumTree. A single-aggregate count secondary would reopen the #809 finding-C proof-relabeling forgery, and a plain Reference folds to (1, 0) in a PCPS tree, silently zeroing the #806 band Total. The count axis therefore carries count_value_as_sum(count) as its payload sum. - Rows bind the IMMEDIATE primary node's committed value hash, not a terminal. That keeps the invariant local and mirror-maintainable. This is dedicated indexed-tree behaviour selected explicitly at each call site — ordinary user references keep their terminal contract and diagnostics, and nothing infers the rule from max_reference_hop == 1. - The secondary has no SubtreePath, so reference resolution for rows is purpose-built machinery keyed on the primary's logical path, not the generic path-keyed follow_reference. Write path: - Mirrors track the primary node's value hash alongside (count, sum), so a value-only update — and equally a deep mutation that only moves a child subtree root — refreshes every configured axis. This is the intended write amplification. - Capture widened from can_mutate_child_count to a new can_mutate_indexed_secondary_row: the non-Merk append ops leave (count, sum) alone but rewrite the entry's commitment. The direct MMR append also mirrors its entry, which the propagation walk cannot see because that mutation lands at the start path. Proof path: - Axis proofs resolve rows and emit reference-aware nodes carrying the referenced primary value. The verifier authenticates the canonical reference metadata by reconstructing the expected row bytes from the secondary-key suffix and the node's aggregates and checking them against the committed reference hash — so a row's target, hop budget and carried sum are checked, not assumed. - New Node::KVRefValueHashCountSumWithTargetChildHash carries a layered target's child commitment. Tree-shaped primaries are the NORMAL case under a count-indexed tree, and the existing ref node family cannot express their combined hash. - The count-offset gap is closed rather than bypassed, on both the indexed-axis path and the generic one: the short-circuit now runs a reference post-pass before encoding, and the prover/verifier rejections are replaced by resolution plus authentication. Integrity verification compares the exact canonical row, the reference target against the key suffix, and the row's committed hash against the primary node's current one, with distinct sentinels for non-canonical shape, wrong target, wrong payload sum and stale commitment. Costs: rows are sized from the real canonical row (they scale with the primary key they reference), and the combined-reference write charges its extra hash. Full workspace test suite passes. --- grovedb-query/src/proofs/encoding.rs | 138 +++ grovedb-query/src/proofs/mod.rs | 44 + grovedb/src/batch/indexed_tree/mirror.rs | 55 +- grovedb/src/batch/indexed_tree/mod.rs | 74 +- grovedb/src/batch/indexed_tree/pre_state.rs | 26 +- grovedb/src/batch/mod.rs | 48 +- .../src/estimated_costs/average_case_costs.rs | 70 +- grovedb/src/lib.rs | 378 +++++++- grovedb/src/operations/indexed_tree.rs | 884 +++++++++++++----- grovedb/src/operations/mmr_tree.rs | 108 ++- grovedb/src/operations/proof/generate.rs | 84 +- .../operations/proof/indexed_axis/generate.rs | 95 +- .../src/operations/proof/indexed_axis/mod.rs | 1 + .../indexed_axis/reference_resolution.rs | 362 +++++++ grovedb/src/operations/proof/mod.rs | 17 + grovedb/src/operations/proof/verify.rs | 39 +- .../src/tests/count_offset_paginated_tests.rs | 70 +- grovedb/src/tests/coverage_misc_tests.rs | 54 +- .../tests/provable_count_sum_tree_tests.rs | 1 + merk/src/merk/chunks.rs | 3 + merk/src/proofs/branch/mod.rs | 6 +- merk/src/proofs/query/count_offset/emit.rs | 34 +- merk/src/proofs/query/count_offset/tests.rs | 2 + merk/src/proofs/query/count_offset/verify.rs | 155 ++- merk/src/proofs/query/verify.rs | 25 + merk/src/proofs/tree.rs | 36 +- 26 files changed, 2352 insertions(+), 457 deletions(-) create mode 100644 grovedb/src/operations/proof/indexed_axis/reference_resolution.rs diff --git a/grovedb-query/src/proofs/encoding.rs b/grovedb-query/src/proofs/encoding.rs index a864ca369..9f8178dc0 100644 --- a/grovedb-query/src/proofs/encoding.rs +++ b/grovedb-query/src/proofs/encoding.rs @@ -540,6 +540,35 @@ impl Encode for Op { sum.encode_into(dest)?; } } + Op::Push(Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + count, + sum, + target_child_hash, + )) => { + let key_len = key_len_u8(key)?; + if value.len() < 65536 { + dest.write_all(&[0x4e, key_len])?; + dest.write_all(key)?; + (value.len() as u16).encode_into(dest)?; + dest.write_all(value)?; + dest.write_all(value_hash)?; + count.encode_into(dest)?; + sum.encode_into(dest)?; + dest.write_all(target_child_hash)?; + } else { + dest.write_all(&[0x4f, key_len])?; + dest.write_all(key)?; + (value.len() as u32).encode_into(dest)?; + dest.write_all(value)?; + dest.write_all(value_hash)?; + count.encode_into(dest)?; + sum.encode_into(dest)?; + dest.write_all(target_child_hash)?; + } + } Op::Push(Node::KVDigestCountSum(key, value_hash, count, sum)) => { let key_len = key_len_u8(key)?; @@ -609,6 +638,35 @@ impl Encode for Op { sum.encode_into(dest)?; } } + Op::PushInverted(Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + count, + sum, + target_child_hash, + )) => { + let key_len = key_len_u8(key)?; + if value.len() < 65536 { + dest.write_all(&[0x50, key_len])?; + dest.write_all(key)?; + (value.len() as u16).encode_into(dest)?; + dest.write_all(value)?; + dest.write_all(value_hash)?; + count.encode_into(dest)?; + sum.encode_into(dest)?; + dest.write_all(target_child_hash)?; + } else { + dest.write_all(&[0x51, key_len])?; + dest.write_all(key)?; + (value.len() as u32).encode_into(dest)?; + dest.write_all(value)?; + dest.write_all(value_hash)?; + count.encode_into(dest)?; + sum.encode_into(dest)?; + dest.write_all(target_child_hash)?; + } + } Op::PushInverted(Node::KVDigestCountSum(key, value_hash, count, sum)) => { let key_len = key_len_u8(key)?; @@ -790,6 +848,23 @@ impl Encode for Op { + count.encoding_length()? + sum.encoding_length()? } + Op::Push(Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + _, + count, + sum, + _, + )) => { + let header = if value.len() < 65536 { 4 } else { 6 }; + header + + key.len() + + value.len() + + HASH_LENGTH + + count.encoding_length()? + + sum.encoding_length()? + + HASH_LENGTH + } Op::Push(Node::KVDigestCountSum(key, _, count, sum)) => { 2 + key.len() + HASH_LENGTH + count.encoding_length()? + sum.encoding_length()? } @@ -817,6 +892,23 @@ impl Encode for Op { + count.encoding_length()? + sum.encoding_length()? } + Op::PushInverted(Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + _, + count, + sum, + _, + )) => { + let header = if value.len() < 65536 { 4 } else { 6 }; + header + + key.len() + + value.len() + + HASH_LENGTH + + count.encoding_length()? + + sum.encoding_length()? + + HASH_LENGTH + } Op::PushInverted(Node::KVDigestCountSum(key, _, count, sum)) => { 2 + key.len() + HASH_LENGTH + count.encoding_length()? + sum.encoding_length()? } @@ -1855,6 +1947,52 @@ impl Decode for Op { sum, )) } + // Resolved reference with a layered target. Same layout as + // `KVRefValueHashCountSum` (0x43/0x44, 0x4a/0x4b) plus a + // trailing 32-byte target child hash. + 0x4e | 0x4f | 0x50 | 0x51 => { + let inverted = matches!(variant, 0x50 | 0x51); + let long_value = matches!(variant, 0x4f | 0x51); + let key_len: u8 = Decode::decode(&mut input)?; + let mut key = vec![0; key_len as usize]; + input.read_exact(key.as_mut_slice())?; + + let value_len: u32 = if long_value { + let len: u32 = Decode::decode(&mut input)?; + if len > MAX_VALUE_LEN { + return Err(ed::Error::UnexpectedByte(variant)); + } + len + } else { + let len: u16 = Decode::decode(&mut input)?; + len as u32 + }; + let mut value = vec![0; value_len as usize]; + input.read_exact(value.as_mut_slice())?; + + let mut value_hash = [0; HASH_LENGTH]; + input.read_exact(&mut value_hash)?; + + let count: u64 = Decode::decode(&mut input)?; + let sum: i64 = Decode::decode(&mut input)?; + + let mut target_child_hash = [0; HASH_LENGTH]; + input.read_exact(&mut target_child_hash)?; + + let node = Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + count, + sum, + target_child_hash, + ); + if inverted { + Self::PushInverted(node) + } else { + Self::Push(node) + } + } 0x10 => Self::Parent, 0x11 => Self::Child, diff --git a/grovedb-query/src/proofs/mod.rs b/grovedb-query/src/proofs/mod.rs index 6898b1361..fb70ee206 100644 --- a/grovedb-query/src/proofs/mod.rs +++ b/grovedb-query/src/proofs/mod.rs @@ -245,6 +245,33 @@ pub enum Node { /// /// Contains: `(kv_hash, left_child_hash, right_child_hash, count, sum)` HashWithCountAndSum(CryptoHash, CryptoHash, CryptoHash, u64, i64), + + /// A resolved reference whose TARGET is itself layered — a subtree, an + /// indexed element, or another reference — so the target's committed + /// value hash is a combined hash rather than `H(target_value)`. + /// + /// `KVRefValueHashCountSum` cannot express these: it reconstructs the + /// row's commitment as + /// `combine_hash(reference_element_hash, H(referenced_value))`, and no + /// choice of `referenced_value` makes `H(referenced_value)` equal a + /// combined hash. This variant carries the missing half — the target's + /// own child/next-hop commitment — so the verifier can rebuild the + /// full chain: + /// + /// ```text + /// target_committed = combine_hash(H(referenced_value), target_child_hash) + /// row_committed = combine_hash(reference_element_hash, target_committed) + /// ``` + /// + /// Introduced for indexed-tree axis proofs, where the rows are + /// canonical references and the primary entries they point at are + /// routinely subtrees (a count-indexed tree indexes its children's + /// counts, so tree-shaped children are the normal case, not an edge + /// case). + /// + /// Contains: `(key, referenced_value, reference_element_hash, count, + /// sum, target_child_hash)` + KVRefValueHashCountSumWithTargetChildHash(Vec, Vec, CryptoHash, u64, i64, CryptoHash), } use std::fmt; @@ -374,6 +401,23 @@ impl fmt::Display for Node { count, sum ), + Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + count, + sum, + target_child_hash, + ) => format!( + "KVRefValueHashCountSumWithTargetChildHash({}, {}, HASH[{}], count={}, sum={}, \ + target_child=HASH[{}])", + hex_to_ascii(key), + hex_to_ascii(value), + hex::encode(value_hash), + count, + sum, + hex::encode(target_child_hash) + ), Node::KVDigestCountSum(key, value_hash, count, sum) => format!( "KVDigestCountSum({}, HASH[{}], count={}, sum={})", hex_to_ascii(key), diff --git a/grovedb/src/batch/indexed_tree/mirror.rs b/grovedb/src/batch/indexed_tree/mirror.rs index 50cfab09a..1582673b6 100644 --- a/grovedb/src/batch/indexed_tree/mirror.rs +++ b/grovedb/src/batch/indexed_tree/mirror.rs @@ -22,9 +22,9 @@ use grovedb_merk::{ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; -use super::{read_entry_aggregates, AggregatePair, AggregateTransition}; +use super::{read_entry_aggregates, AggregateTransition, MaybeEntryState}; use crate::{ - operations::indexed_tree::{axis_row_payload, make_axis_secondary_key}, + operations::indexed_tree::{axis_row_reference, make_axis_secondary_key}, Element, Error, }; @@ -63,7 +63,7 @@ fn enforce_axis_item_key_bound<'a>( /// what makes each axis's assembled batch deterministic. pub(crate) fn read_post_apply_transitions<'db, S: StorageContext<'db>>( primary_merk: &Merk, - pre: &BTreeMap, AggregatePair>, + pre: &BTreeMap, MaybeEntryState>, grove_version: &GroveVersion, ) -> CostResult, Error> { let mut cost = OperationCost::default(); @@ -80,10 +80,18 @@ pub(crate) fn read_post_apply_transitions<'db, S: StorageContext<'db>>( /// Assemble one axis's secondary row moves into a sorted Merk batch. /// -/// Each transition is compared as this axis's `(key, payload)` rather than -/// the raw aggregates: on the avg axis two different `(count, sum)` pairs -/// can share a sort key while carrying different payloads, and on the count -/// axis a sum change moves nothing at all. +/// Each transition is compared as this axis's `(key, row, target hash)` +/// rather than the raw aggregates: on the avg axis two different +/// `(count, sum)` pairs can share a sort key while carrying different +/// payloads, on the count axis a sum change moves nothing at all, and a +/// canonical reference row is stale whenever the primary node's committed +/// value hash moves even if nothing about the sort position changed. +/// +/// That last case is the intentional write amplification the reference +/// representation buys: a value-only primary update — and equally a deep +/// mutation that only changes a child subtree's root, or a +/// `RefreshReference` on a reference-shaped primary — now rewrites every +/// configured axis's row. fn build_axis_mirror_batch( transitions: &[AggregateTransition], axis: IndexAxis, @@ -92,19 +100,20 @@ fn build_axis_mirror_batch( let mut cost = OperationCost::default(); let secondary_tree_type = crate::operations::indexed_tree::axis_secondary_tree_type(axis); let mut secondary_batch: Vec>> = Vec::with_capacity(transitions.len() * 2); - for (key, old_aggregates, new_aggregates) in transitions { - let entry_for = |aggregates: &AggregatePair| -> Result<_, Error> { - aggregates - .map(|(c, s)| { + for (key, old_state, new_state) in transitions { + let entry_for = |state: &MaybeEntryState| -> Result<_, Error> { + state + .map(|s| { Ok(( - make_axis_secondary_key(axis, c, s, key), - axis_row_payload(axis, c, s)?, + make_axis_secondary_key(axis, s.count, s.sum, key), + axis_row_reference(axis, key, s.count, s.sum)?, + s.value_hash, )) }) .transpose() }; - let old_entry = cost_return_on_error_no_add!(cost, entry_for(old_aggregates)); - let new_entry = cost_return_on_error_no_add!(cost, entry_for(new_aggregates)); + let old_entry = cost_return_on_error_no_add!(cost, entry_for(old_state)); + let new_entry = cost_return_on_error_no_add!(cost, entry_for(new_state)); if old_entry == new_entry { continue; } @@ -115,8 +124,8 @@ fn build_axis_mirror_batch( // Merk batch is rejected outright ("Keys in batch must be unique"), // failing the whole GroveDB batch. Where the key is unchanged the put // alone overwrites the payload, which is what the row needs. - let new_secondary_key_ref = new_entry.as_ref().map(|(key, _)| key); - if let Some((old_secondary_key, _)) = &old_entry + let new_secondary_key_ref = new_entry.as_ref().map(|(key, ..)| key); + if let Some((old_secondary_key, ..)) = &old_entry && Some(old_secondary_key) != new_secondary_key_ref { cost_return_on_error!( @@ -131,18 +140,26 @@ fn build_axis_mirror_batch( .map_err(Error::MerkError) ); } - if let Some((new_secondary_key, entry)) = new_entry { + if let Some((new_secondary_key, entry, target_value_hash)) = new_entry { let feature_type = cost_return_on_error_no_add!( cost, entry .get_feature_type(secondary_tree_type) .map_err(Error::MerkError) ); + // `PutCombinedReference`, not a plain put: the row's committed + // value hash must be + // `combine_hash(H(reference bytes), target_value_hash)` so the + // secondary root binds the primary entry it points at. A plain + // put would commit only the reference bytes, leaving a row that + // authenticates its own path while saying nothing about the + // value at the other end of it. cost_return_on_error!( &mut cost, entry - .insert_into_batch_operations( + .insert_reference_into_batch_operations( new_secondary_key, + target_value_hash, &mut secondary_batch, feature_type, grove_version, diff --git a/grovedb/src/batch/indexed_tree/mod.rs b/grovedb/src/batch/indexed_tree/mod.rs index fdccd0929..4c5fee5a6 100644 --- a/grovedb/src/batch/indexed_tree/mod.rs +++ b/grovedb/src/batch/indexed_tree/mod.rs @@ -55,7 +55,7 @@ pub(crate) use delete_tree::validate_delete_tree_type; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{element::costs::ElementCostExtensions, Merk}; +use grovedb_merk::{element::costs::ElementCostExtensions, CryptoHash, Merk}; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; pub(crate) use mirror::{apply_indexed_secondary_mirror_post_apply, read_post_apply_transitions}; @@ -65,22 +65,30 @@ pub(crate) use preflight::reject_indexed_overwrite_with_descendants; use crate::{Element, Error}; -/// A primary entry's `(count, sum)` aggregate pair, `None` when the entry -/// does not exist on that side of the transition. -type AggregatePair = Option<(u64, i64)>; +use crate::operations::indexed_tree::IndexedEntryState; -/// One captured key's aggregate transition: `(item_key, old, new)`. -type AggregateTransition = (Vec, AggregatePair, AggregatePair); +/// A primary entry's state, `None` when the entry does not exist on that +/// side of the transition. +type MaybeEntryState = Option; -/// Read one primary entry's current `(count, sum)` pair, `None` if the key -/// does not exist. `phase` labels error messages ("pre" for the capture -/// before the primary apply, "post" for the delta read after it). +/// One captured key's transition: `(item_key, old, new)`. +type AggregateTransition = (Vec, MaybeEntryState, MaybeEntryState); + +/// Read one primary entry's current state, `None` if the key does not +/// exist. `phase` labels error messages ("pre" for the capture before the +/// primary apply, "post" for the delta read after it). +/// +/// Reads the element and the node's value hash. The value hash is taken +/// from the Merk node rather than recomputed from the serialized bytes: +/// for a tree-shaped or reference-shaped entry the stored hash is a +/// combined hash that `value_hash(bytes)` cannot reproduce, and it is the +/// stored one the row must bind. fn read_entry_aggregates<'db, S: StorageContext<'db>>( primary_merk: &Merk, key: &[u8], phase: &str, grove_version: &GroveVersion, -) -> CostResult { +) -> CostResult { let mut cost = OperationCost::default(); let maybe_bytes = cost_return_on_error!( &mut cost, @@ -96,16 +104,40 @@ fn read_entry_aggregates<'db, S: StorageContext<'db>>( hex::encode(key) ))) ); - let aggregates = if let Some(bytes) = maybe_bytes { - let elem = cost_return_on_error_no_add!( - cost, - Element::deserialize(bytes.as_slice(), grove_version).map_err(|e| { - Error::CorruptedData(format!("indexed {phase}-state deserialize: {e}")) - }) - ); - Some(elem.count_sum_value_or_default()) - } else { - None + let Some(bytes) = maybe_bytes else { + return Ok(None).wrap_with_cost(cost); }; - Ok(aggregates).wrap_with_cost(cost) + let elem = cost_return_on_error_no_add!( + cost, + Element::deserialize(bytes.as_slice(), grove_version) + .map_err(|e| Error::CorruptedData(format!("indexed {phase}-state deserialize: {e}"))) + ); + let value_hash = cost_return_on_error!( + &mut cost, + primary_merk + .get_value_hash( + key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|e| Error::CorruptedData(format!( + "indexed {phase}-state value hash for key {}: {e}", + hex::encode(key) + ))) + ); + let value_hash = cost_return_on_error_no_add!( + cost, + value_hash.ok_or_else(|| Error::CorruptedData(format!( + "indexed {phase}-state: key {} has element bytes but no node value hash", + hex::encode(key) + ))) + ); + let (count, sum) = elem.count_sum_value_or_default(); + Ok(Some(IndexedEntryState { + count, + sum, + value_hash, + })) + .wrap_with_cost(cost) } diff --git a/grovedb/src/batch/indexed_tree/pre_state.rs b/grovedb/src/batch/indexed_tree/pre_state.rs index 2f342d8cd..3fff5921e 100644 --- a/grovedb/src/batch/indexed_tree/pre_state.rs +++ b/grovedb/src/batch/indexed_tree/pre_state.rs @@ -14,7 +14,7 @@ use grovedb_merk::{element::insert::ElementInsertToStorageExtensions, Merk}; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; -use super::{read_entry_aggregates, AggregatePair}; +use super::{read_entry_aggregates, MaybeEntryState}; use crate::{ batch::{GroveOp, KeyInfo}, operations::indexed_tree::MAX_CIDX_ITEM_KEY_LEN, @@ -150,13 +150,16 @@ fn validate_indexed_child_ops( /// the avg axis derives its sort key from the pair, and a PCPSIT can index /// count, sum and avg simultaneously. /// -/// Only ops whose `can_mutate_child_count()` is true are captured — -/// non-count-mutating ops (e.g., `CommitmentTreeInsert`) are skipped. +/// Every op that can rewrite the entry is captured — see +/// `GroveOp::can_mutate_indexed_secondary_row`. That is wider than +/// count-mutation: a canonical row binds the primary node's commitment, +/// so a non-Merk append (which leaves `(count, sum)` alone but writes a +/// new root into the entry) must be mirrored too. pub(crate) fn capture_indexed_pre_state<'db, S: StorageContext<'db>>( primary_merk: &Merk, ops_at_path_by_key: &BTreeMap, grove_version: &GroveVersion, -) -> CostResult, AggregatePair>, Error> { +) -> CostResult, MaybeEntryState>, Error> { let mut cost = OperationCost::default(); cost_return_on_error_no_add!(cost, enforce_indexed_item_key_ceiling(ops_at_path_by_key)); @@ -165,15 +168,16 @@ pub(crate) fn capture_indexed_pre_state<'db, S: StorageContext<'db>>( validate_indexed_child_ops(ops_at_path_by_key, primary_merk.tree_type) ); - let mut pre: BTreeMap, AggregatePair> = BTreeMap::new(); + let mut pre: BTreeMap, MaybeEntryState> = BTreeMap::new(); for (key_info, op) in ops_at_path_by_key.iter() { let key_bytes = key_info.get_key_clone(); - // Single source of truth: `GroveOp::can_mutate_child_count` - // uses an exhaustive match so adding a new variant forces - // explicit classification at the type-system level. This is - // the structural guard against the nested-cidx bug class - // (commit a8bb34fb). - if op.can_mutate_child_count() && !pre.contains_key(&key_bytes) { + // Single source of truth: + // `GroveOp::can_mutate_indexed_secondary_row` uses an exhaustive + // match so adding a new variant forces explicit classification at + // the type-system level. This is the structural guard against the + // nested-cidx bug class (commit a8bb34fb) and, since rows became + // references, against the unmirrored-commitment class too. + if op.can_mutate_indexed_secondary_row() && !pre.contains_key(&key_bytes) { let old_aggregates = cost_return_on_error!( &mut cost, read_entry_aggregates(primary_merk, &key_bytes, "pre", grove_version) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index e01228bd3..7a8f608ba 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -577,6 +577,48 @@ impl GroveOp { | GroveOp::DenseTreeInsert { .. } => false, } } + + /// Whether an op can move an indexed primary entry's canonical + /// secondary ROW — a strictly wider question than + /// [`Self::can_mutate_child_count`]. + /// + /// A canonical row binds the primary node's committed value hash as + /// well as its `(count, sum)`, so any op that rewrites the entry at + /// all can staleness the row. The non-Merk append ops are exactly the + /// difference: an `MmrTreeAppend` leaves `(count, sum)` untouched but + /// writes a new non-Merk root into the entry, which moves its + /// commitment. Capturing on `can_mutate_child_count` left those rows + /// bound to a hash that no longer existed, which `verify_grovedb` + /// then reported as a stale target. + /// + /// EXHAUSTIVE on purpose — no `_` arm — so a new op variant is a + /// compile error here rather than a silently unmirrored mutation. + /// That is the same technique `can_mutate_child_count` uses, for the + /// same bug class. + pub(crate) fn can_mutate_indexed_secondary_row(&self) -> bool { + match self { + GroveOp::InsertWithKnownToNotAlreadyExist { .. } + | GroveOp::InsertIfNotExists { .. } + | GroveOp::InsertOrReplace { .. } + | GroveOp::Replace { .. } + | GroveOp::Patch { .. } + | GroveOp::Delete + | GroveOp::DeleteTree(..) + | GroveOp::RefreshReference { .. } + | GroveOp::ReplaceTreeRootKey { .. } + | GroveOp::InsertTreeWithRootHash { .. } + | GroveOp::ReplaceNonMerkTreeRoot { .. } + | GroveOp::InsertNonMerkTree { .. } + | GroveOp::ReplaceAggregateIndexedTreeRootKeys { .. } + | GroveOp::InsertAggregateIndexedTreeRootKeys { .. } + // The four that `can_mutate_child_count` excludes: they + // rewrite the entry's non-Merk root, hence its commitment. + | GroveOp::CommitmentTreeInsert { .. } + | GroveOp::MmrTreeAppend { .. } + | GroveOp::BulkAppend { .. } + | GroveOp::DenseTreeInsert { .. } => true, + } + } } impl PartialOrd for GroveOp { @@ -2386,9 +2428,9 @@ where // primary level represent a child subtree's bubble-up — the // child's element bytes have a new aggregate count, so its // secondary entry needs to move; we capture it here too. - let indexed_pre_state: Option, Option<(u64, i64)>>> = if in_tree_type - .is_indexed_primary() - { + let indexed_pre_state: Option< + BTreeMap, Option>, + > = if in_tree_type.is_indexed_primary() { let merk = self.merks.get(path).expect("the Merk is cached"); Some(cost_return_on_error!( &mut cost, diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 4182a717d..7f339078a 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -481,15 +481,24 @@ impl GroveDb { /// sparse or conditional indexing). /// - **Key size** is the primary's key size plus the axis sort-key width /// (8 bytes for count/sum, 16 for avg). - /// - **Value size** is bounded by the fixed per-axis payload shape: - /// a `SumItem` (count and sum axes), or an empty - /// `ItemWithSumItem` (avg) — all under - /// [`INDEXED_SECONDARY_MAX_VALUE_SIZE`]. + /// - **Value size** is the canonical reference row's own serialized + /// size. Unlike the fixed-shape placeholder rows this replaced, a + /// reference row carries the primary key it points at, so its size + /// scales with the primary's key size — it is derived from the real + /// row rather than pinned to a constant. /// - **Tree type** is fixed per axis. /// /// Each axis is charged one Merk open, one delete of the old row and one /// insert of the new row, each propagating — which is what /// `mirror_*_to_secondary` actually performs. + /// + /// **Reference refresh is charged unconditionally.** A canonical row + /// binds the primary node's commitment, so a value-only primary update + /// — one that moves neither count nor sum — still rewrites every + /// configured axis. The estimate must reflect that: it is an admission + /// bound replayed against historical blocks, and an estimate that + /// assumed "aggregates unchanged ⇒ no secondary write" would come in + /// under actual `added_bytes`. pub fn average_case_indexed_secondary_mirror( primary_path: &KeyInfoPath, primary_layer_information: &EstimatedLayerInformation, @@ -508,13 +517,38 @@ impl GroveDb { for axis in axes { let secondary_key_size = (primary_key_size + axis_sort_key_len(*axis) as u32).min(u8::MAX as u32) as u8; + // The worst-case canonical row for this axis, sized against a + // primary key of the estimated width. Built through THE row + // function the mirror writes with, so the estimate cannot drift + // from the write path: sum values are charged at their fixed + // worst-case varint width, so `i64::MAX` is an upper bound rather + // than an average (and is inside `count_value_as_sum`'s domain, + // so the conversion cannot fail here). + let worst_case_row = cost_return_on_error_no_add!( + cost, + crate::operations::indexed_tree::axis_row_reference( + *axis, + &vec![0u8; primary_key_size as usize], + i64::MAX as u64, + i64::MAX, + ) + ); + let row_value_size = cost_return_on_error_no_add!( + cost, + worst_case_row + .serialized_size(grove_version) + .map_err(|e| Error::CorruptedData(format!( + "sizing the worst-case indexed secondary row: {e}" + ))) + ) + .min(u32::MAX as usize) as u32; let secondary_layer = EstimatedLayerInformation { tree_type: axis_secondary_tree_type(*axis), // 1:1 with the primary. estimated_layer_count: primary_layer_information.estimated_layer_count, estimated_layer_sizes: EstimatedLayerSizes::AllItems( secondary_key_size, - INDEXED_SECONDARY_MAX_VALUE_SIZE, + row_value_size, None, ), }; @@ -540,21 +574,17 @@ impl GroveDb { // The mirror is delete-old-row then insert-new-row, each of which // rebalances and re-roots the secondary. // - // The inserted row must be sized with the AXIS's real payload - // shape: `average_case_merk_insert_element` charges non-tree - // elements by their own serialized size, and under-sizing put - // the PCPSIT estimate ~25 bytes per key UNDER actual - // `added_bytes` — the one dimension a storage-fee reservation - // cannot come in under. The shape comes from THE payload - // function the mirror writes with (`axis_row_payload`), fed - // worst-case aggregates: sum values are charged at their fixed - // worst-case varint width, so `i64::MAX` is the upper bound, - // not an average (and is in `count_value_as_sum`'s domain, so - // the conversion cannot fail here). - let worst_case_row = cost_return_on_error_no_add!( - cost, - crate::operations::indexed_tree::axis_row_payload(*axis, i64::MAX as u64, i64::MAX,) - ); + // The inserted row must be sized with the AXIS's real row shape: + // `average_case_merk_insert_element` charges non-tree elements by + // their own serialized size, and under-sizing put the PCPSIT + // estimate ~25 bytes per key UNDER actual `added_bytes` — the one + // dimension a storage-fee reservation cannot come in under. + // + // Writing the row is one extra hash beyond a plain put: a + // combined reference commits + // `combine_hash(H(row bytes), target hash)`, so the value hash is + // hashed twice rather than once. + cost.hash_node_calls += 1; cost_return_on_error!( &mut cost, Self::average_case_merk_delete_element( diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a035fd440..8d0942145 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -736,6 +736,137 @@ impl GroveDb { ) } + /// Mirror ONE entry of an indexed primary into every axis it + /// configures, returning each axis's post-mirror `(tag, root_hash, + /// root_key)` so the caller can hand it to propagation as the initial + /// deferred state. + /// + /// This exists for the direct non-Merk append APIs (MMR, commitment, + /// bulk-append, dense). Those write the updated element straight into + /// the primary Merk and only then start propagating, so the + /// propagation walk — which mirrors entries it discovers as it climbs + /// — never sees the entry that actually moved. That was harmless while + /// rows were aggregate-only (a non-Merk child's count is a constant + /// `1`, so its row never moved) and is not harmless now: a canonical + /// row binds the primary node's commitment, and an append rewrites + /// exactly that. + /// + /// `old_state` must be captured BEFORE the entry was rewritten. + pub(crate) fn mirror_indexed_primary_entry<'b, B: AsRef<[u8]>>( + &self, + primary_path: SubtreePath<'b, B>, + primary_merk: &Merk, + indexed_element: &Element, + item_key: &[u8], + old_state: Option, + transaction: &Transaction, + batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult>)>, Error> { + use crate::operations::indexed_tree::{ + mirror_indexed_axis_to_secondary, IndexedEntryState, + }; + + let mut cost = OperationCost::default(); + + let axes: Vec<(u8, Option>)> = match indexed_element.underlying() { + Element::ProvableCountIndexedTree(_, secondary, ..) => { + vec![( + grovedb_element::indexed::IndexAxis::Count.tag(), + secondary.clone(), + )] + } + Element::ProvableSumIndexedTree(_, secondary, ..) => { + vec![( + grovedb_element::indexed::IndexAxis::Sum.tag(), + secondary.clone(), + )] + } + Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => axes.clone(), + _ => { + return Err(Error::CorruptedCodeExecution( + "mirror_indexed_primary_entry called with a non-indexed element", + )) + .wrap_with_cost(cost); + } + }; + + // Post-write state of the entry, read from the primary Merk the + // caller already updated. + let new_state = { + let maybe_element = cost_return_on_error!( + &mut cost, + Element::get_optional(primary_merk, item_key, true, grove_version) + .map_err(Error::MerkError) + ); + match maybe_element { + None => None, + Some(element) => { + let value_hash = cost_return_on_error!( + &mut cost, + primary_merk + .get_value_hash( + item_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let (count, sum) = element.count_sum_value_or_default(); + value_hash.map(|value_hash| IndexedEntryState { + count, + sum, + value_hash, + }) + } + } + }; + + let mut staged: Vec<(u8, Hash, Option>)> = Vec::with_capacity(axes.len()); + for (tag, secondary_root_key) in axes { + let axis = cost_return_on_error_no_add!( + cost, + grovedb_element::indexed::IndexAxis::try_from_tag(tag).map_err(|e| { + Error::CorruptedData(format!( + "invalid axis tag on an indexed element during entry mirror: {e}" + )) + }) + ); + let mut secondary_merk = cost_return_on_error!( + &mut cost, + self.open_indexed_secondary_at_path( + primary_path.clone(), + axis, + secondary_root_key, + transaction, + Some(batch), + grove_version, + ) + ); + cost_return_on_error!( + &mut cost, + mirror_indexed_axis_to_secondary( + &mut secondary_merk, + axis, + item_key, + old_state, + new_state, + grove_version, + ) + ); + let (sec_hash, sec_key, _) = cost_return_on_error!( + &mut cost, + secondary_merk + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) + ); + staged.push((tag, sec_hash, sec_key)); + } + + Ok(staged).wrap_with_cost(cost) + } + pub(crate) fn propagate_changes_with_transaction_with_initial_deferred<'b, B: AsRef<[u8]>>( &self, mut merk_cache: HashMap, Merk>, @@ -744,6 +875,37 @@ impl GroveDb { transaction: &Transaction, batch: &StorageBatch, grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + self.propagate_changes_with_transaction_with_initial_deferred_axes( + merk_cache, + path, + initial_deferred_secondary, + None, + transaction, + batch, + grove_version, + ) + } + + /// As [`Self::propagate_changes_with_transaction_with_initial_deferred`], + /// but also seeds the MULTI-AXIS deferred state a PCPSIT primary needs. + /// A single-axis seed cannot express a PCPSIT's per-axis roots, so the + /// direct non-Merk append APIs — which mirror their entry themselves + /// via [`Self::mirror_indexed_primary_entry`] before propagating — go + /// through here. + #[allow(clippy::too_many_arguments)] + pub(crate) fn propagate_changes_with_transaction_with_initial_deferred_axes< + 'b, + B: AsRef<[u8]>, + >( + &self, + mut merk_cache: HashMap, Merk>, + path: SubtreePath<'b, B>, + initial_deferred_secondary: Option<(Hash, Option>)>, + initial_deferred_axes: Option>)>>, + transaction: &Transaction, + batch: &StorageBatch, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { use grovedb_merk::element::{ get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, @@ -795,7 +957,7 @@ impl GroveDb { // Re-reading the axes from the element instead would open each // secondary by its stale pre-mirror root key and rebuild a digest // over the old state. - let mut deferred_axes: Option>)>> = None; + let mut deferred_axes: Option>)>> = initial_deferred_axes; while let Some((parent_path, parent_key)) = current_path.derive_parent() { let mut parent_tree: Merk = cost_return_on_error!( @@ -816,18 +978,40 @@ impl GroveDb { let parent_is_indexed_primary = parent_tree.tree_type.is_indexed_primary(); - // Snapshot the old ordering values of the element at parent_key - // BEFORE we mutate parent_tree. We need them later to compute the - // per-axis delta for secondary mirroring. `count_and_sum` covers - // every axis: Count orders on the count, Sum on the sum, and Avg - // on the fixed-point ratio of the two. + // Snapshot the old state of the element at parent_key BEFORE we + // mutate parent_tree. We need it later to compute the per-axis + // delta for secondary mirroring. `count_and_sum` covers every + // axis: Count orders on the count, Sum on the sum, and Avg on the + // fixed-point ratio of the two. The value hash rides along + // because a canonical secondary row binds the primary node's + // commitment, so an unchanged `(count, sum)` no longer implies an + // unchanged row — which is precisely the case this propagation + // path hits, since it runs when a child subtree's root moved. let old_ordering_in_parent = if parent_is_indexed_primary { let old_element = cost_return_on_error!( &mut cost, Element::get(&parent_tree, parent_key, true, grove_version) .map_err(Error::MerkError) ); - Some(old_element.count_sum_value_or_default()) + let old_value_hash = cost_return_on_error!( + &mut cost, + parent_tree + .get_value_hash( + parent_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let (count, sum) = old_element.count_sum_value_or_default(); + old_value_hash.map(|value_hash| { + crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, + } + }) } else { None }; @@ -1013,7 +1197,7 @@ impl GroveDb { // count delta into its secondary and stage the new secondary // state for the NEXT iteration (which will reach the element // that holds primary_root_key and secondary_root_key). - if let Some((old_count, old_sum)) = old_ordering_in_parent { + if let Some(old_state) = old_ordering_in_parent { // Take the new ordering values from the element we just wrote, // NOT from `aggregate_data`. The batch path reads both sides // of the delta from the element @@ -1031,6 +1215,35 @@ impl GroveDb { .map_err(Error::MerkError) ); let (new_count, new_sum) = new_element_in_parent.count_sum_value_or_default(); + // Read the POST-write commitment: this is the hash the + // canonical row must bind, and it is exactly what moved when + // the child subtree's root changed. + let new_value_hash = cost_return_on_error!( + &mut cost, + parent_tree + .get_value_hash( + parent_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let new_state = cost_return_on_error_no_add!( + cost, + new_value_hash + .map(|value_hash| { + crate::operations::indexed_tree::IndexedEntryState { + count: new_count, + sum: new_sum, + value_hash, + } + }) + .ok_or(Error::CorruptedCodeExecution( + "indexed primary entry has element bytes but no node value hash \ + immediately after being written during propagation", + )) + ); // The indexed element carrying the secondary root key(s) lives // one level up, at grandparent[indexed_key]. @@ -1120,10 +1333,8 @@ impl GroveDb { &mut secondary_merk, axis, parent_key, - Some(old_count), - Some(old_sum), - Some(new_count), - Some(new_sum), + Some(old_state), + Some(new_state), grove_version, ) ); @@ -1672,9 +1883,18 @@ impl GroveDb { slot }; - // Expected secondary key for every primary entry, derived from the - // entry's own aggregates exactly as the mirror derives it. - let mut expected: HashMap, (Vec, Element)> = HashMap::new(); + // Expected secondary key, canonical row, and bound target hash for + // every primary entry, derived exactly as the mirror derives them. + // + // Indexed rows are validated HERE, in the dedicated indexed context — + // never routed through the generic `verify_references` arm. That arm + // follows references to their TERMINAL and combines against the + // terminal's hash, which is deliberately a different binding from the + // immediate-primary-node rule canonical rows use. Ordinary user + // references keep their terminal contract and diagnostics untouched; + // the one-hop immediate-node rule is selected explicitly here and + // nowhere else. + let mut expected: HashMap, (Vec, Element, Option)> = HashMap::new(); let mut content_iter = KVIterator::new(primary_merk.storage.raw_iter(), &all_query).unwrap(); while let Some((p_key, p_value)) = content_iter.next_kv().unwrap() { @@ -1695,13 +1915,37 @@ impl GroveDb { } let p_elem = Element::raw_decode(&p_value, grove_version)?; let (count, sum) = p_elem.count_sum_value_or_default(); - // The payload the mirror writes for this axis, so a row filed - // under the right key but carrying the wrong value is caught too - // — the key alone does not pin the stored aggregate. - let payload = crate::operations::indexed_tree::axis_row_payload(axis, count, sum)?; + // The row the mirror writes for this axis, so a row filed under + // the right key but carrying the wrong reference or the wrong + // carried sum is caught too — the key alone pins neither. + let row = + crate::operations::indexed_tree::axis_row_reference(axis, &p_key, count, sum)?; + // The immediate primary node's committed value hash — read from + // the node, not recomputed from `p_value`, because for a tree- or + // reference-shaped entry the stored hash is a combined hash that + // `value_hash(bytes)` cannot reproduce. + // + // `None` means the raw iterator found bytes the AVL cannot reach + // from the committed root — an unlinked row. That is corruption in + // its own right, and it is reported by the shape checks below; we + // just cannot state a commitment for it, so the commitment + // comparison is skipped rather than turned into a hard error that + // would mask every other finding in the walk. + let target_value_hash = primary_merk + .get_value_hash( + &p_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap()?; expected.insert( p_key.clone(), - (make_axis_secondary_key(axis, count, sum, &p_key), payload), + ( + make_axis_secondary_key(axis, count, sum, &p_key), + row, + target_value_hash, + ), ); } drop(content_iter); @@ -1711,7 +1955,8 @@ impl GroveDb { // real drift class: the same item in two sort buckets) are visible // instead of silently collapsing. let sort_len = axis_sort_key_len(axis); - let mut actual: HashMap, Vec<(Vec, Element)>> = HashMap::new(); + let mut actual: HashMap, Vec<(Vec, Element, Option)>> = + HashMap::new(); let mut sec_iter = KVIterator::new(secondary_merk.storage.raw_iter(), &all_query).unwrap(); while let Some((sec_key, sec_value)) = sec_iter.next_kv().unwrap() { if sec_key.len() < sort_len { @@ -1722,11 +1967,29 @@ impl GroveDb { continue; } let sec_elem = Element::raw_decode(&sec_value, grove_version)?; + // The row's COMMITTED value hash: for a canonical combined + // reference this is `combine_hash(H(ref bytes), target hash)`, so + // it is what proves the row still binds the primary entry's + // current state. Recomputing it from `sec_value` alone would only + // re-derive the reference bytes' own hash and see nothing stale. + // `None` here means the row is present in storage but unreachable + // from the committed root — an unlinked row, which the shape and + // orphan checks below already surface. Skip its commitment + // comparison rather than hard-erroring the whole walk. + let stored_value_hash = secondary_merk + .get_value_hash( + &sec_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap()?; let item_key = sec_key[sort_len..].to_vec(); - actual - .entry(item_key) - .or_default() - .push((sec_key.clone(), sec_elem)); + actual.entry(item_key).or_default().push(( + sec_key.clone(), + sec_elem, + stored_value_hash, + )); } drop(sec_iter); @@ -1742,7 +2005,7 @@ impl GroveDb { } } - for (item_key, (want_key, want_payload)) in &expected { + for (item_key, (want_key, want_row, want_target_hash)) in &expected { match actual.get(item_key).and_then(|v| v.first()) { None => { let mut p = new_path.to_vec(); @@ -1750,7 +2013,7 @@ impl GroveDb { p.push(item_key.clone()); issues.insert(p, ([0u8; 32], [0u8; 32], [0u8; 32])); } - Some((got_key, _)) if got_key != want_key => { + Some((got_key, ..)) if got_key != want_key => { // The item is indexed, but under the wrong sort key — // i.e. the secondary's ordering value is stale. Report // the DECODED ordering value (right-aligned in the hash @@ -1768,16 +2031,35 @@ impl GroveDb { ), ); } - Some((_, got_payload)) if got_payload != want_payload => { - // Filed under the right key, but the stored payload - // disagrees with what the primary implies — the key - // encodes the ordering value, not the stored aggregate, - // so this is invisible to a key-only comparison. + Some((_, got_row, _)) if got_row != want_row => { + // Filed under the right key, but the stored row disagrees + // with what the primary implies — the key encodes the + // ordering value, not the reference or its carried sum, + // so this is invisible to a key-only comparison. Split + // into distinct sentinels so a corruption report says + // WHICH part of the row is wrong. + let describe = "verify_grovedb indexed row"; + let kind = match crate::operations::indexed_tree::decode_axis_row_reference( + got_row, describe, + ) { + // Not a canonical row shape at all: a legacy + // placeholder payload, a plain `Reference`, a + // non-sibling reference, or a wrong hop budget. + Err(_) => "secondary_non_canonical_row", + // Canonical shape, but pointing at the wrong primary + // key. + Ok((target, _)) if target != item_key.as_slice() => { + "secondary_wrong_reference_target" + } + // Canonical shape and target, so what differs is the + // carried sum (the axis payload sum). + Ok(_) => "secondary_wrong_payload_sum", + }; let mut p = new_path.to_vec(); - p.push(sentinel("secondary_value_mismatch")); + p.push(sentinel(kind)); p.push(item_key.clone()); - let want_bytes = want_payload.serialize(grove_version)?; - let got_bytes = got_payload.serialize(grove_version)?; + let want_bytes = want_row.serialize(grove_version)?; + let got_bytes = got_row.serialize(grove_version)?; issues.insert( p, ( @@ -1787,7 +2069,31 @@ impl GroveDb { ), ); } - Some(_) => { /* indexed under the expected sort key and value */ } + Some((_, got_row, got_stored_hash)) => { + // Right key, right row bytes — but the row's COMMITMENT + // can still be stale: a value-only primary update moves + // the target hash while leaving key and bytes identical. + // That is the exact drift the reference representation + // exists to make detectable, so it gets its own sentinel. + // + // Only checkable when both sides are reachable from their + // committed roots; an unlinked row on either side is + // reported by the shape/orphan checks instead. + if let (Some(want_target_hash), Some(got_stored_hash)) = + (want_target_hash, got_stored_hash) + { + let row_bytes = got_row.serialize(grove_version)?; + let want_committed = + combine_hash(&value_hash(&row_bytes).unwrap(), want_target_hash) + .unwrap(); + if &want_committed != got_stored_hash { + let mut p = new_path.to_vec(); + p.push(sentinel("secondary_stale_target_hash")); + p.push(item_key.clone()); + issues.insert(p, ([0u8; 32], want_committed, *got_stored_hash)); + } + } + } } } diff --git a/grovedb/src/operations/indexed_tree.rs b/grovedb/src/operations/indexed_tree.rs index 8edfe00a0..9f8fa1412 100644 --- a/grovedb/src/operations/indexed_tree.rs +++ b/grovedb/src/operations/indexed_tree.rs @@ -104,35 +104,169 @@ pub(crate) fn count_value_as_sum(count: u64) -> Result { }) } -/// THE per-axis secondary row payload — the single definition every -/// writer and every checker uses. The sort KEY encodes the ordering -/// value; this payload carries what the secondary's own dual -/// aggregates must fold to: +/// Hop budget stamped on every canonical indexed-secondary row. /// -/// - Count → `SumItem(count_value)` — contributes `(1, count)`, so a -/// band TOTAL is one committed scalar (issue #806) -/// - Sum → `SumItem(sum)` — contributes `(1, sum)` -/// - Avg → `ItemWithSumItem(empty, sum)` — contributes `(1, sum)` +/// The row binds the IMMEDIATE primary node, not a terminal: its +/// committed value hash is +/// `combine_hash(H(canonical_reference_bytes), primary_node_value_hash)` +/// where `primary_node_value_hash` is whatever the primary Merk stores +/// for that key — a simple hash for an item, a layered/combined hash for +/// a tree or a nested reference. Binding the immediate node is what keeps +/// the invariant LOCAL and therefore mirror-maintainable: a mutation to +/// some distant terminal cannot staleness this row without also rewriting +/// the primary entry, which is the event the mirror is driven by. /// -/// Callers: the batch mirror row builder, the direct-path mirror, the -/// reconcile repair loop, `verify_grovedb`'s expected-payload check, -/// and the average-case cost estimator's worst-case row. They MUST all -/// go through this function: the mirror writes these bytes into -/// hash-committed state and the checkers recompute them independently, -/// so a divergent copy at any site either false-flags healthy state or -/// makes two entry points commit different root hashes for identical -/// writes. (This function exists because exactly that drift risk was -/// flagged by the #809 security audit.) +/// This is dedicated indexed-tree behaviour and is NOT a relaxation of +/// ordinary user-reference semantics — an ordinary `max_hop = 1` +/// reference pointing at another reference remains ill-formed and keeps +/// its existing diagnostics. Every consumer of an indexed row must select +/// the immediate-node rule explicitly (that is what +/// [`resolve_indexed_row_target`] and [`verify_indexed_axis_content`] do); +/// nothing may infer it from `max_reference_hop == 1` alone. +/// +/// [`resolve_indexed_row_target`]: crate::GroveDb::resolve_indexed_row_target +/// [`verify_indexed_axis_content`]: crate::GroveDb +pub(crate) const INDEXED_SECONDARY_MAX_HOP: grovedb_element::MaxReferenceHop = Some(1); + +/// One primary entry's mirror-relevant state. +/// +/// The aggregates decide the row's sort key and carried sum; the value +/// hash decides its reference commitment. Every mirror compares both +/// sides of a transition as all three, because a canonical row binds the +/// primary node — an entry whose value changed while its `(count, sum)` +/// stayed put still needs its row rewritten, which is exactly the case +/// the pre-reference mirror was free to skip. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct IndexedEntryState { + /// The entry's count aggregate. + pub(crate) count: u64, + /// The entry's sum aggregate. + pub(crate) sum: i64, + /// The primary node's Merk-stored committed value hash — a simple + /// hash for an item, a layered/combined hash for a tree, a combined + /// hash for a nested reference. This is the immediate-node binding + /// target; it is NOT resolved through to a terminal (see + /// [`INDEXED_SECONDARY_MAX_HOP`]). + pub(crate) value_hash: grovedb_merk::CryptoHash, +} + +/// The sum an axis's canonical row carries — the axis PAYLOAD sum, which +/// is not universally the primary's sum: +/// +/// - Count → `count_value_as_sum(count)`, so a band TOTAL over the count +/// axis stays one committed scalar (issue #806). A plain `Reference` +/// here would fold to `(1, 0)` and silently zero every band total. +/// - Sum / Avg → the primary entry's own sum. +/// +/// Every writer and every checker must agree on this one definition; a +/// divergent copy either false-flags healthy state or makes two entry +/// points commit different roots for identical writes (#809 audit). /// /// Fallible only through [`count_value_as_sum`]'s fail-closed guard. -pub(crate) fn axis_row_payload(axis: IndexAxis, count: u64, sum: i64) -> Result { +#[inline] +pub(crate) fn axis_payload_sum(axis: IndexAxis, count: u64, sum: i64) -> Result { Ok(match axis { - IndexAxis::Count => Element::new_sum_item(count_value_as_sum(count)?), - IndexAxis::Sum => Element::new_sum_item(sum), - IndexAxis::Avg => Element::new_item_with_sum_item(Vec::new(), sum), + IndexAxis::Count => count_value_as_sum(count)?, + IndexAxis::Sum | IndexAxis::Avg => sum, }) } +/// THE per-axis secondary row — the single definition every writer and +/// every checker uses. The sort KEY encodes the ordering value; the row +/// itself is a canonical one-hop reference back to the primary entry, +/// carrying the axis payload sum so the secondary's dual aggregates fold +/// to `(1, axis_payload_sum)`. +/// +/// All three axes share one element family: +/// `ReferenceWithSumItem(SiblingReference(item_key), Some(1), sum)`. +/// +/// The `SiblingReference` is interpreted against the row's LOGICAL +/// origin — the indexed primary's path — not against the derived storage +/// prefix the secondary physically lives under (which is not a GroveDB +/// path at all). See [`indexed_row_target_key`] for the decoding side and +/// [`crate::operations::indexed_tree`]'s module docs for the origin rule. +/// +/// Callers: the batch mirror row builder, the direct-path mirror, the +/// propagation mirror, `verify_grovedb`'s expected-row check, the axis +/// proof generator, and the average-case cost estimator's worst-case row. +/// +/// Fallible only through [`count_value_as_sum`]'s fail-closed guard. +pub(crate) fn axis_row_reference( + axis: IndexAxis, + item_key: &[u8], + count: u64, + sum: i64, +) -> Result { + Ok(Element::new_reference_with_sum_item_with_hops( + grovedb_element::reference_path::ReferencePathType::SiblingReference(item_key.to_vec()), + INDEXED_SECONDARY_MAX_HOP, + axis_payload_sum(axis, count, sum)?, + )) +} + +/// Decode a stored secondary row, enforcing the canonical shape and +/// returning `(target_item_key, carried_sum)`. +/// +/// Rejects anything that is not exactly +/// `ReferenceWithSumItem(SiblingReference(_), Some(1), _)` — including +/// the legacy placeholder payloads (`SumItem` / `ItemWithSumItem`), a +/// plain `Reference` (which would fold to `(1, 0)`), a non-sibling +/// reference type, and a wrong hop budget. `describe` labels the caller +/// in the error so a corruption report says where it was caught. +pub(crate) fn decode_axis_row_reference<'a>( + row: &'a Element, + describe: &str, +) -> Result<(&'a [u8], i64), Error> { + match row { + Element::ReferenceWithSumItem(reference_path, max_hop, sum, _) => { + if *max_hop != INDEXED_SECONDARY_MAX_HOP { + return Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row carries max_reference_hop {max_hop:?}, \ + canonical rows are one-hop ({INDEXED_SECONDARY_MAX_HOP:?})" + ))); + } + match reference_path { + grovedb_element::reference_path::ReferencePathType::SiblingReference(key) => { + Ok((key.as_slice(), *sum)) + } + other => Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row must be a SiblingReference to its \ + primary entry, found {other}" + ))), + } + } + other => Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row must be ReferenceWithSumItem, found {}", + other.type_str() + ))), + } +} + +/// The primary item key a canonical row points at, cross-checked against +/// the item key its own secondary-key suffix encodes. +/// +/// The suffix and the reference path are two independent encodings of the +/// same fact, and a proof or verification pass that trusts one without +/// the other accepts a row whose committed reference points somewhere the +/// sort position does not. Both the verifier and the proof generator go +/// through here so neither can drift into checking only one. +pub(crate) fn indexed_row_target_key<'a>( + row: &'a Element, + key_suffix_item_key: &[u8], + describe: &str, +) -> Result<(&'a [u8], i64), Error> { + let (target, sum) = decode_axis_row_reference(row, describe)?; + if target != key_suffix_item_key { + return Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row references {} but its secondary-key suffix \ + encodes {}", + hex::encode(target), + hex::encode(key_suffix_item_key) + ))); + } + Ok((target, sum)) +} + /// Build the secondary key bytes for an entry at `item_key` under the /// given axis, given the relevant aggregate values: /// - count axis → `count_be(8) ‖ item_key` @@ -1018,13 +1152,14 @@ impl GroveDb { .min() .expect("every indexed variant carries at least one axis"); - // 3. Walk the primary once, collecting each entry's (count, sum) - // pair — every axis derives its rows from the same pair. + // 3. Walk the primary once, collecting each entry's state — every + // axis derives its rows from the same `(count, sum)` pair, and + // every axis binds the same primary node commitment. let mut all_query = Query::new(); all_query.insert_all(); let mut iter = KVIterator::new(primary_merk.storage.raw_iter(), &all_query).unwrap_add_cost(&mut cost); - let mut entries: Vec<(Vec, (u64, i64))> = Vec::new(); + let mut entries: Vec<(Vec, IndexedEntryState)> = Vec::new(); while let Some((key, value_bytes)) = iter.next_kv().unwrap_add_cost(&mut cost) { // Reject oversized primary keys before they can drive // make_axis_secondary_key to synthesize a secondary key that @@ -1052,7 +1187,35 @@ impl GroveDb { )) }) ); - entries.push((key, element.count_sum_value_or_default())); + let value_hash = cost_return_on_error!( + &mut cost, + primary_merk + .get_value_hash( + key.as_slice(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|e| Error::CorruptedData(format!( + "reading primary node value hash while reconciling secondary: {e}" + ))) + ); + let value_hash = cost_return_on_error_no_add!( + cost, + value_hash.ok_or_else(|| Error::CorruptedData(format!( + "primary entry {} has element bytes but no node value hash", + hex::encode(&key) + ))) + ); + let (count, sum) = element.count_sum_value_or_default(); + entries.push(( + key, + IndexedEntryState { + count, + sum, + value_hash, + }, + )); } // 4. Rebuild each axis's secondary and capture its post-repair @@ -1074,27 +1237,32 @@ impl GroveDb { ); let secondary_tree_type = axis_secondary_tree_type(axis); - // Desired rows: key AND serialized payload. `BTreeMap`, not a - // hashed map: the repair loops below iterate it, and a hashed - // iteration order would build the secondary AVL in a different - // shape on every run — two operators repairing identical - // databases must derive identical secondary root hashes, or the - // H1-A parent bindings would disagree. - let mut desired: std::collections::BTreeMap, Vec> = - std::collections::BTreeMap::new(); - for (key, (count, sum)) in &entries { - let secondary_key = make_axis_secondary_key(axis, *count, *sum, key); - let payload = - cost_return_on_error_no_add!(cost, axis_row_payload(axis, *count, *sum)); - let payload_bytes = cost_return_on_error_no_add!( + // Desired rows: key, serialized canonical row, and the primary + // node commitment the row must bind. `BTreeMap`, not a hashed + // map: the repair loops below iterate it, and a hashed iteration + // order would build the secondary AVL in a different shape on + // every run — two operators repairing identical databases must + // derive identical secondary root hashes, or the H1-A parent + // bindings would disagree. + let mut desired: std::collections::BTreeMap< + Vec, + (Vec, grovedb_merk::CryptoHash), + > = std::collections::BTreeMap::new(); + for (key, state) in &entries { + let secondary_key = make_axis_secondary_key(axis, state.count, state.sum, key); + let row = cost_return_on_error_no_add!( cost, - payload.serialize(grove_version).map_err(|e| { + axis_row_reference(axis, key, state.count, state.sum) + ); + let row_bytes = cost_return_on_error_no_add!( + cost, + row.serialize(grove_version).map_err(|e| { Error::CorruptedData(format!( - "failed to serialize desired secondary payload: {e}" + "failed to serialize desired secondary row: {e}" )) }) ); - desired.insert(secondary_key, payload_bytes); + desired.insert(secondary_key, (row_bytes, state.value_hash)); } // Existing row KEYS, raw-iterated so unlinked-but-present rows @@ -1129,10 +1297,12 @@ impl GroveDb { } } - // Insert missing rows and rewrite payload-damaged ones. Sum and - // avg rows carry real payloads, so key-presence alone does not - // imply row-correctness; compare the stored element bytes. - for (desired_key, desired_payload) in &desired { + // Insert missing rows and rewrite damaged ones. Key-presence + // alone does not imply row-correctness on any axis: the row + // carries a reference path and a payload sum the key does not + // encode, and its COMMITMENT can be stale even when every byte + // of the row is right. Compare all three. + for (desired_key, (desired_row_bytes, desired_target_hash)) in &desired { let needs_write = if existing_keys.contains(desired_key) { let stored = cost_return_on_error!( &mut cost, @@ -1144,20 +1314,40 @@ impl GroveDb { grove_version, ) .map_err(|e| Error::CorruptedData(format!( - "reading secondary row for payload compare: {e}" + "reading secondary row for compare: {e}" + ))) + ); + let stored_value_hash = cost_return_on_error!( + &mut cost, + secondary_merk + .get_value_hash( + desired_key.as_slice(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|e| Error::CorruptedData(format!( + "reading secondary row commitment for compare: {e}" ))) ); - stored.as_deref() != Some(desired_payload.as_slice()) + let want_committed = grovedb_merk::tree::combine_hash( + &grovedb_merk::tree::value_hash(desired_row_bytes) + .unwrap_add_cost(&mut cost), + desired_target_hash, + ) + .unwrap_add_cost(&mut cost); + stored.as_deref() != Some(desired_row_bytes.as_slice()) + || stored_value_hash != Some(want_committed) } else { true }; if needs_write { let entry = cost_return_on_error_no_add!( cost, - Element::deserialize(desired_payload.as_slice(), grove_version).map_err( + Element::deserialize(desired_row_bytes.as_slice(), grove_version).map_err( |e| { Error::CorruptedData(format!( - "failed to round-trip desired secondary payload: {e}" + "failed to round-trip desired secondary row: {e}" )) } ) @@ -1165,9 +1355,10 @@ impl GroveDb { cost_return_on_error!( &mut cost, entry - .insert( + .insert_reference( &mut secondary_merk, desired_key.as_slice(), + *desired_target_hash, None, grove_version, ) @@ -2454,83 +2645,63 @@ impl GroveDb { } } -/// Apply a PCPSIT axis secondary mirror covering insert, update, and -/// delete via the (old, new) Option pair. Reads `old_count`/`old_sum` -/// from the prior primary state and `new_count`/`new_sum` from the -/// post-mutation state. +/// Apply an axis secondary mirror covering insert, update, and delete via +/// the (old, new) [`IndexedEntryState`] pair. /// -/// The row payload is [`axis_row_payload`] — see its doc for the -/// per-axis shapes and why every writer must share it. -#[allow(clippy::too_many_arguments)] +/// The row is [`axis_row_reference`] — a canonical one-hop +/// `ReferenceWithSumItem` back to the primary entry — written as a +/// COMBINED reference so the secondary root binds +/// `combine_hash(H(reference bytes), primary_node_value_hash)`. pub(crate) fn mirror_indexed_axis_to_secondary<'db, S: StorageContext<'db>>( secondary: &mut Merk, axis: IndexAxis, item_key: &[u8], - old_count: Option, - old_sum: Option, - new_count: Option, - new_sum: Option, + old: Option, + new: Option, grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let secondary_tree_type = axis_secondary_tree_type(axis); - // The payload is a function of BOTH aggregates (the count axis - // mirrors count_value into its sum half), so the fast-path equality - // check below must see the full pair — a sum-only view would - // silently skip real payload updates. - let axis_payload = |count: u64, sum: i64| axis_row_payload(axis, count, sum); - - // Compute old and new sort keys. Either may be None (no entry). - let old_key = match (old_count, old_sum) { - (Some(c), Some(s)) => Some(make_axis_secondary_key(axis, c, s, item_key)), - _ => None, - }; - let new_key = match (new_count, new_sum) { - (Some(c), Some(s)) => Some(make_axis_secondary_key(axis, c, s, item_key)), - _ => None, + let row_for = |state: Option| -> Result<_, Error> { + state + .map(|s| { + Ok(( + make_axis_secondary_key(axis, s.count, s.sum, item_key), + axis_row_reference(axis, item_key, s.count, s.sum)?, + s.value_hash, + )) + }) + .transpose() }; - - // Fast path: skip the delete+insert only when BOTH the sort key AND - // the stored payload are unchanged. - // - // Equal sort keys do NOT imply equal payloads on the Avg axis: the - // avg key is floor(sum * 10^19 / count) while the payload carries the - // raw `sum`, so e.g. (count, sum) = (1, 5) and (2, 10) share a key - // (avg 5.0) but differ in payload sum (5 vs 10). Returning early on - // key-equality alone would leave a stale hash-committed sum in the - // avg secondary. On the Count / Sum axes the payload is a pure - // function of what the key already encodes, so this reduces to the - // former key-only check and preserves the fast path for the common - // case. - if old_key == new_key && new_key.is_some() { - let payload_unchanged = match ((old_count, old_sum), (new_count, new_sum)) { - ((Some(oc), Some(os)), (Some(nc), Some(ns))) => { - cost_return_on_error_no_add!(cost, axis_payload(oc, os)) - == cost_return_on_error_no_add!(cost, axis_payload(nc, ns)) - } - _ => false, - }; - if payload_unchanged { - // The sort key didn't move and the stored value is identical; - // the previous insert already wrote the correct value, so - // nothing more is needed. - return Ok(()).wrap_with_cost(cost); - } + let old_entry = cost_return_on_error_no_add!(cost, row_for(old)); + let new_entry = cost_return_on_error_no_add!(cost, row_for(new)); + + // Fast path: skip only when the sort key, the row bytes AND the bound + // target hash are all unchanged. The target hash is what makes this + // strictly narrower than the pre-reference check: a value-only primary + // update leaves key and row identical while moving the commitment, and + // skipping it would leave a row authenticating a value that is no + // longer there. + if old_entry == new_entry { + return Ok(()).wrap_with_cost(cost); } - // A payload change at a FIXED key (avg axis only: a proportional - // (count, sum) change keeps the average) must replace in place. - // Deleting and reinserting the same key rebalances the AVL twice - // and can settle a DIFFERENT shape than the batch mirror's single - // replacement write — two write paths committing different - // secondary (hence grove) root hashes for identical data, which + // A row change at a FIXED key must replace in place. Deleting and + // reinserting the same key rebalances the AVL twice and can settle a + // DIFFERENT shape than the batch mirror's single replacement write — + // two write paths committing different secondary (hence grove) root + // hashes for identical data, which // `direct_and_batch_agree_on_root_for_a_fixed_key_avg_payload_change` - // reproduced on an interior node before this skip existed. The - // insert below overwrites the value in place, exactly like the - // batch path's put. - let key_moved = old_key != new_key; - if let (true, Some(ok)) = (key_moved, &old_key) { + // reproduced on an interior node before this skip existed. The insert + // below overwrites the value in place, exactly like the batch path's + // put. + let old_key = old_entry.as_ref().map(|(k, ..)| k); + let new_key = new_entry.as_ref().map(|(k, ..)| k); + if let Some(ok) = old_key + && Some(ok) != new_key + { + let ok = ok.clone(); cost_return_on_error!( &mut cost, Element::delete( @@ -2544,21 +2715,17 @@ pub(crate) fn mirror_indexed_axis_to_secondary<'db, S: StorageContext<'db>>( .map_err(Error::MerkError) ); } - if let (Some(nk), Some(new_count_val), Some(new_sum_val)) = (&new_key, new_count, new_sum) { - // Derived by the same `axis_payload` closure the fast-path - // equality check above uses, so the two stay in lockstep. - // Every axis's secondary is a dual-aggregate - // ProvableCountProvableSumTree: - // - Count → SumItem(count_value) — contributes (1, count), so a - // band TOTAL is one committed scalar (issue #806) - // - Sum → SumItem(sum) — contributes (1, sum) - // - Avg → ItemWithSumItem(empty, sum) — contributes (1, sum) - let entry = cost_return_on_error_no_add!(cost, axis_payload(new_count_val, new_sum_val)); + if let Some((nk, row, target_value_hash)) = new_entry { cost_return_on_error!( &mut cost, - entry - .insert(secondary, nk.as_slice(), None, grove_version) - .map_err(Error::MerkError) + row.insert_reference( + secondary, + nk.as_slice(), + target_value_hash, + None, + grove_version + ) + .map_err(Error::MerkError) ); } Ok(()).wrap_with_cost(cost) @@ -3191,80 +3358,186 @@ impl GroveDb { } #[cfg(test)] -mod axis_row_payload_tests { - //! The payload function is THE definition every writer and checker +mod axis_row_reference_tests { + //! The row function is THE definition every writer and checker //! shares; this grid pins its output per (axis, count, sum) so any //! change to the shape is a deliberate, reviewed event — the bytes //! land in hash-committed state, so an accidental change here means //! mirrors and checkers disagree about healthy databases. - use grovedb_element::indexed::IndexAxis; + use grovedb_element::{indexed::IndexAxis, reference_path::ReferencePathType}; use grovedb_version::version::GroveVersion; - use super::axis_row_payload; + use super::{ + axis_payload_sum, axis_row_reference, decode_axis_row_reference, indexed_row_target_key, + INDEXED_SECONDARY_MAX_HOP, + }; use crate::Element; + fn sibling(key: &[u8], sum: i64) -> Element { + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(key.to_vec()), + INDEXED_SECONDARY_MAX_HOP, + sum, + ) + } + #[test] - fn payload_grid_is_pinned_per_axis() { + fn row_grid_is_pinned_per_axis() { let counts = [0u64, 1, 2, i64::MAX as u64]; let sums = [i64::MIN, -1, 0, 1, i64::MAX]; + let key = b"item".as_slice(); for &count in &counts { for &sum in &sums { assert_eq!( - axis_row_payload(IndexAxis::Count, count, sum).unwrap(), - Element::new_sum_item(count as i64), - "count axis stores the COUNT as its sum item; the sum input is ignored" + axis_row_reference(IndexAxis::Count, key, count, sum).unwrap(), + sibling(key, count as i64), + "count axis carries the COUNT as its payload sum; the sum input is ignored" ); assert_eq!( - axis_row_payload(IndexAxis::Sum, count, sum).unwrap(), - Element::new_sum_item(sum), - "sum axis stores the sum; the count input is ignored" + axis_row_reference(IndexAxis::Sum, key, count, sum).unwrap(), + sibling(key, sum), + "sum axis carries the sum; the count input is ignored" ); assert_eq!( - axis_row_payload(IndexAxis::Avg, count, sum).unwrap(), - Element::new_item_with_sum_item(Vec::new(), sum), - "avg axis stores an empty item carrying the sum" + axis_row_reference(IndexAxis::Avg, key, count, sum).unwrap(), + sibling(key, sum), + "avg axis carries the sum, exactly as the sum axis does" ); } } // Above the sum-item domain the count axis fails closed. - axis_row_payload(IndexAxis::Count, i64::MAX as u64 + 1, 0) + axis_row_reference(IndexAxis::Count, key, i64::MAX as u64 + 1, 0) + .expect_err("count above i64::MAX must fail closed"); + axis_payload_sum(IndexAxis::Count, i64::MAX as u64 + 1, 0) .expect_err("count above i64::MAX must fail closed"); } + #[test] + fn every_axis_uses_one_canonical_element_family() { + // Locked decision 2: one element family across all three axes. A + // plain `Reference` on the count axis would fold to (1, 0) in a + // PCPS secondary and silently zero every band Total (#806), and a + // single-aggregate secondary would reopen the #809 finding-C proof + // relabeling. Both regressions start by this assertion failing. + for axis in [IndexAxis::Count, IndexAxis::Sum, IndexAxis::Avg] { + let row = axis_row_reference(axis, b"k", 3, 5).unwrap(); + assert!( + matches!(row, Element::ReferenceWithSumItem(..)), + "{axis:?} row must be ReferenceWithSumItem, got {}", + row.type_str() + ); + assert_eq!( + super::axis_secondary_tree_type(axis), + grovedb_merk::TreeType::ProvableCountProvableSumTree, + "{axis:?} secondary must stay dual-aggregate" + ); + } + // The count axis's payload sum is the count, not the primary sum. + assert_eq!(axis_payload_sum(IndexAxis::Count, 3, 5).unwrap(), 3); + assert_eq!(axis_payload_sum(IndexAxis::Sum, 3, 5).unwrap(), 5); + assert_eq!(axis_payload_sum(IndexAxis::Avg, 3, 5).unwrap(), 5); + } + + #[test] + fn decode_round_trips_and_rejects_non_canonical_rows() { + let row = axis_row_reference(IndexAxis::Sum, b"target", 1, 42).unwrap(); + assert_eq!( + decode_axis_row_reference(&row, "test").unwrap(), + (b"target".as_slice(), 42) + ); + assert_eq!( + indexed_row_target_key(&row, b"target", "test").unwrap(), + (b"target".as_slice(), 42) + ); + + // Suffix disagreement: the row's reference and its sort-key suffix + // are two independent encodings of the same fact, and trusting one + // without the other accepts a row whose commitment points somewhere + // its sort position does not. + indexed_row_target_key(&row, b"other", "test") + .expect_err("reference target must agree with the secondary-key suffix"); + + // Legacy placeholder rows must be rejected outright. + for legacy in [ + Element::new_sum_item(7), + Element::new_item_with_sum_item(Vec::new(), 7), + Element::new_item(Vec::new()), + ] { + decode_axis_row_reference(&legacy, "test") + .expect_err("legacy placeholder rows are not valid indexed rows"); + } + // A plain `Reference` folds to (1, 0) in a PCPS secondary — it must + // not be accepted as a canonical row. + decode_axis_row_reference( + &Element::new_reference(ReferencePathType::SiblingReference(b"target".to_vec())), + "test", + ) + .expect_err("a plain Reference carries no payload sum and is not canonical"); + // Non-sibling reference types would make row size grow with grove + // depth and break the logical-origin rule. + decode_axis_row_reference( + &Element::new_reference_with_sum_item_with_hops( + ReferencePathType::AbsolutePathReference(vec![b"a".to_vec()]), + INDEXED_SECONDARY_MAX_HOP, + 7, + ), + "test", + ) + .expect_err("only SiblingReference is canonical"); + // Wrong hop budget: the binding rule is one hop to the immediate + // primary node, and a different budget means a different binding. + decode_axis_row_reference( + &Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"target".to_vec()), + Some(2), + 7, + ), + "test", + ) + .expect_err("canonical rows are one-hop"); + decode_axis_row_reference( + &Element::new_reference_with_sum_item( + ReferencePathType::SiblingReference(b"target".to_vec()), + 7, + ), + "test", + ) + .expect_err("an unbounded hop budget is not canonical"); + } + #[test] fn serialized_bytes_are_stable() { // The exact bytes the mirror hash-commits, pinned as FIXED // vectors — not re-serialized at assertion time, so a // serialization-format change cannot move both sides of the - // comparison and slip through. If this test fails, payload - // bytes in authenticated state have changed: that is a - // consensus event, not a refactor. + // comparison and slip through. If this test fails, row bytes in + // authenticated state have changed: that is a consensus event, + // not a refactor. let grove_version = GroveVersion::latest(); assert_eq!( - axis_row_payload(IndexAxis::Count, 7, 0) + axis_row_reference(IndexAxis::Count, b"k", 7, 0) .unwrap() .serialize(grove_version) .unwrap(), - vec![3, 14, 0], - "count axis: SumItem(7) as [variant, zigzag-varint 7, no flags]" + vec![18, 6, 1, 107, 1, 1, 14, 0], + "count axis: ReferenceWithSumItem(SiblingReference('k'), hop 1, sum 7)" ); assert_eq!( - axis_row_payload(IndexAxis::Sum, 1, -3) + axis_row_reference(IndexAxis::Sum, b"k", 1, -3) .unwrap() .serialize(grove_version) .unwrap(), - vec![3, 5, 0], - "sum axis: SumItem(-3) as [variant, zigzag-varint -3, no flags]" + vec![18, 6, 1, 107, 1, 1, 5, 0], + "sum axis: ReferenceWithSumItem(SiblingReference('k'), hop 1, sum -3)" ); assert_eq!( - axis_row_payload(IndexAxis::Avg, 1, 5) + axis_row_reference(IndexAxis::Avg, b"k", 1, 5) .unwrap() .serialize(grove_version) .unwrap(), - vec![9, 0, 10, 0], - "avg axis: ItemWithSumItem(empty, 5) as [variant, empty item, \ - zigzag-varint 5, no flags]" + vec![18, 6, 1, 107, 1, 1, 10, 0], + "avg axis: ReferenceWithSumItem(SiblingReference('k'), hop 1, sum 5)" ); } } @@ -3415,42 +3688,88 @@ mod secondary_key_codec_tests { } } } - #[cfg(test)] -mod bug2_avg_axis_mirror_tests { - //! BUG 2 regression: `mirror_indexed_axis_to_secondary` must not - //! early-return on the Avg axis when the sort key is unchanged but - //! the stored payload sum differs. +mod direct_axis_mirror_tests { + //! Direct-drive tests for `mirror_indexed_axis_to_secondary`. //! - //! The avg sort key is `floor(sum * 10^19 / count)`, while the stored - //! payload is `ItemWithSumItem(_, sum)`. Two `(count, sum)` pairs can - //! share a key yet differ in payload sum — e.g. `(1, 5)` and `(2, 10)` - //! both encode avg `5.0` but carry payload sums `5` and `10`. The old - //! key-only early-return left the stale hash-committed sum `5` in the - //! avg secondary. + //! Two regression families live here: //! - //! This path is currently unreachable through the public dedicated - //! APIs (each child contributes count 0 or 1, so the mirror never sees - //! a `(1, 5) -> (2, 10)` transition for one item key), so we drive the - //! module-private mirror function directly against a real Avg - //! secondary Merk. + //! **BUG 2** — the mirror must not early-return on the Avg axis when the + //! sort key is unchanged but the carried sum differs. The avg sort key is + //! `floor(sum * 10^19 / count)` while the row carries the raw `sum`, so + //! `(1, 5)` and `(2, 10)` share a key yet carry sums `5` and `10`. The old + //! key-only early-return left the stale hash-committed `5` behind. + //! + //! **Commitment refresh** — a canonical row binds the primary node's + //! committed value hash, so a transition whose key AND carried sum are + //! both unchanged still has to rewrite the row when that hash moves. This + //! is the case a `(count, sum)`-only mirror is structurally blind to, and + //! it is reachable in production through a value-only update, a deep + //! mutation that changes a child subtree's root, and a `RefreshReference` + //! on a reference-shaped primary. + //! + //! Both drive the module-private mirror directly against a real secondary + //! Merk: the transitions involved are not all reachable through the + //! public dedicated APIs (each child contributes count 0 or 1, so no + //! public path produces a `(1, 5) -> (2, 10)` move for one item key). use grovedb_costs::OperationCost; use grovedb_element::indexed::{compute_avg_fixed_point, IndexAxis}; - use grovedb_merk::{element::get::ElementFetchFromStorageExtensions, tree::AggregateData}; + use grovedb_merk::{ + element::{costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions}, + tree::AggregateData, + }; use grovedb_path::SubtreePath; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; - use super::{make_axis_secondary_key, mirror_indexed_axis_to_secondary}; + use super::{ + axis_row_reference, make_axis_secondary_key, mirror_indexed_axis_to_secondary, + IndexedEntryState, + }; use crate::{ tests::{make_test_grovedb, TEST_LEAF}, - Element, + Element, GroveDb, }; + /// A distinct stand-in commitment per byte, so a test can move the bound + /// target hash without needing a real primary entry behind it. + fn target_hash(seed: u8) -> grovedb_merk::CryptoHash { + [seed; 32] + } + + fn state(count: u64, sum: i64, seed: u8) -> IndexedEntryState { + IndexedEntryState { + count, + sum, + value_hash: target_hash(seed), + } + } + + /// Set up a PCPSIT with a single configured axis and hand back an open, + /// empty secondary Merk for it plus the pieces that must outlive it. + fn open_axis_secondary<'db>( + db: &'db GroveDb, + axis: IndexAxis, + grove_version: &GroveVersion, + ) -> (crate::Transaction<'db>, StorageBatch) { + let axes: Vec<(u8, Option>)> = vec![(axis.tag(), None)]; + db.insert( + [TEST_LEAF].as_ref(), + b"pcpsit", + Element::empty_provable_count_provable_sum_indexed_tree(axes).expect("canonical axes"), + None, + None, + grove_version, + ) + .unwrap() + .expect("empty PCPSIT insert"); + (db.start_transaction(), StorageBatch::new()) + } + /// Sanity: `(1, 5)` and `(2, 10)` share an avg sort key but produce - /// different payload sums — the precondition that made the old - /// key-only early-return unsound. + /// different carried sums — the precondition that made the old key-only + /// early-return unsound. #[test] fn avg_key_collision_with_distinct_payload_sums() { let item_key = b"row"; @@ -3465,37 +3784,24 @@ mod bug2_avg_axis_mirror_tests { compute_avg_fixed_point(10, 2), "avg fixed points must match" ); - // Payloads differ (the hash-committed sum the mirror stores). + // The rows differ in their carried sum — the hash-committed value the + // mirror stores. assert_ne!( - Element::new_item_with_sum_item(Vec::new(), 5), - Element::new_item_with_sum_item(Vec::new(), 10), - "payload sums 5 and 10 must differ" + axis_row_reference(IndexAxis::Avg, item_key, 1, 5).unwrap(), + axis_row_reference(IndexAxis::Avg, item_key, 2, 10).unwrap(), + "carried sums 5 and 10 must produce different rows" ); } /// Drive the mirror directly: first write `(1, 5)`, then transition to - /// `(2, 10)` (same avg key). The stored avg-secondary payload sum must - /// update to `10`; the old key-only early-return would have left the - /// stale `5`. + /// `(2, 10)` (same avg key). The stored carried sum must update to `10`; + /// the old key-only early-return would have left the stale `5`. #[test] fn avg_axis_mirror_updates_stale_payload_when_key_unchanged() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); - // Establish a PCPSIT with an Avg axis so the secondary namespace - // is valid. - let axes: Vec<(u8, Option>)> = vec![(IndexAxis::Avg.tag(), None)]; - db.insert( - [TEST_LEAF].as_ref(), - b"pcpsit", - Element::empty_provable_count_provable_sum_indexed_tree(axes).expect("canonical axes"), - None, - None, - grove_version, - ) - .unwrap() - .expect("empty PCPSIT insert"); + let (tx, batch) = open_axis_secondary(&db, IndexAxis::Avg, grove_version); - let tx = db.start_transaction(); let item_key = b"row"; let shared_key = make_axis_secondary_key(IndexAxis::Avg, 1, 5, item_key); assert_eq!( @@ -3504,10 +3810,6 @@ mod bug2_avg_axis_mirror_tests { "test setup: keys must collide" ); - // Open the Avg secondary (empty) and drive the mirror twice - // against the same in-memory Merk. A StorageBatch is required so - // the merk's Element::insert can stage its commit_batch. - let batch = StorageBatch::new(); let path_segments: [&[u8]; 2] = [TEST_LEAF, b"pcpsit".as_ref()]; let path: SubtreePath<_> = (&path_segments).into(); let mut secondary = db @@ -3522,15 +3824,12 @@ mod bug2_avg_axis_mirror_tests { .unwrap() .expect("open empty avg secondary"); - // 1) Insert (count=1, sum=5). mirror_indexed_axis_to_secondary( &mut secondary, IndexAxis::Avg, item_key, None, - None, - Some(1), - Some(5), + Some(state(1, 5, 0xAA)), grove_version, ) .unwrap() @@ -3542,18 +3841,15 @@ mod bug2_avg_axis_mirror_tests { assert_eq!( after_first.sum_value_or_default(), 5, - "payload sum after (1,5) must be 5" + "carried sum after (1,5) must be 5" ); - // 2) Transition to (count=2, sum=10). Same avg key, new sum. mirror_indexed_axis_to_secondary( &mut secondary, IndexAxis::Avg, item_key, - Some(1), - Some(5), - Some(2), - Some(10), + Some(state(1, 5, 0xAA)), + Some(state(2, 10, 0xBB)), grove_version, ) .unwrap() @@ -3565,33 +3861,125 @@ mod bug2_avg_axis_mirror_tests { assert_eq!( after_second.sum_value_or_default(), 10, - "payload sum must update to 10 even though the avg sort key did not \ + "carried sum must update to 10 even though the avg sort key did not \ move (BUG 2 regression: old key-only early-return left stale 5)" ); } - /// The fast path must still short-circuit when BOTH the key and the - /// payload are unchanged (a genuine no-op transition). We verify the - /// stored payload is untouched for an identical (count, sum) rewrite. + /// The commitment-refresh case: key unchanged, carried sum unchanged, but + /// the primary node's value hash moved. The row bytes are identical, so + /// only the stored value hash can show the difference — and it must. + /// + /// This is the transition a `(count, sum)`-only mirror cannot see, and it + /// is exactly what a value-only primary update produces. #[test] - fn avg_axis_mirror_noop_when_key_and_payload_unchanged() { + fn mirror_refreshes_the_row_when_only_the_target_hash_moves() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); - let axes: Vec<(u8, Option>)> = vec![(IndexAxis::Avg.tag(), None)]; - db.insert( - [TEST_LEAF].as_ref(), - b"pcpsit", - Element::empty_provable_count_provable_sum_indexed_tree(axes).expect("canonical axes"), - None, + let (tx, batch) = open_axis_secondary(&db, IndexAxis::Count, grove_version); + + let item_key = b"row"; + let path_segments: [&[u8]; 2] = [TEST_LEAF, b"pcpsit".as_ref()]; + let path: SubtreePath<_> = (&path_segments).into(); + let mut secondary = db + .open_indexed_secondary_at_path( + path, + IndexAxis::Count, + None, + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open empty count secondary"); + + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, item_key); + + mirror_indexed_axis_to_secondary( + &mut secondary, + IndexAxis::Count, + item_key, None, + Some(state(1, 0, 0x11)), grove_version, ) .unwrap() - .expect("empty PCPSIT insert"); + .expect("insert"); + let (root_before, ..) = secondary + .root_hash_key_and_aggregate_data() + .unwrap() + .expect("secondary root state"); + let committed_before = secondary + .get_value_hash( + key.as_slice(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("value hash read") + .expect("row present"); + + // Same count, same sum, same row bytes — only the bound commitment + // moves, as it does when a primary entry's value changes without + // touching its aggregates. + mirror_indexed_axis_to_secondary( + &mut secondary, + IndexAxis::Count, + item_key, + Some(state(1, 0, 0x11)), + Some(state(1, 0, 0x22)), + grove_version, + ) + .unwrap() + .expect("refresh"); + + let committed_after = secondary + .get_value_hash( + key.as_slice(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("value hash read") + .expect("row present"); + let (root_after, ..) = secondary + .root_hash_key_and_aggregate_data() + .unwrap() + .expect("secondary root state"); + + assert_ne!( + committed_before, committed_after, + "a moved target hash must move the row's committed value hash — \ + otherwise the row still authenticates a value that is no longer there" + ); + assert_ne!( + root_before, root_after, + "the refreshed commitment must reach the secondary root, or the \ + staleness never becomes visible to a proof" + ); + // The row BYTES are unchanged: this drift is invisible to any check + // that compares serialized rows alone. + let row = Element::get(&secondary, key.as_slice(), true, grove_version) + .unwrap() + .expect("row present"); + assert_eq!( + row, + axis_row_reference(IndexAxis::Count, item_key, 1, 0).unwrap(), + "the row bytes must be the canonical ones, unchanged by the refresh" + ); + } + + /// The fast path must still short-circuit for a genuine no-op — key, + /// carried sum AND bound target hash all unchanged. + #[test] + fn avg_axis_mirror_noop_when_key_payload_and_target_unchanged() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + let (tx, batch) = open_axis_secondary(&db, IndexAxis::Avg, grove_version); - let tx = db.start_transaction(); let item_key = b"row"; - let batch = StorageBatch::new(); let path_segments: [&[u8]; 2] = [TEST_LEAF, b"pcpsit".as_ref()]; let path: SubtreePath<_> = (&path_segments).into(); let mut secondary = db @@ -3611,15 +3999,13 @@ mod bug2_avg_axis_mirror_tests { IndexAxis::Avg, item_key, None, - None, - Some(2), - Some(10), + Some(state(2, 10, 0x33)), grove_version, ) .unwrap() .expect("insert (2,10)"); - // Identical (count, sum) rewrite: key AND payload unchanged. + // Identical transition: key, row AND target hash unchanged. // Assert on the COST, not just the stored value — a delete+reinsert // would leave the same value behind, so a value-only assertion passes // even with the fast path deleted entirely. @@ -3628,10 +4014,8 @@ mod bug2_avg_axis_mirror_tests { &mut secondary, IndexAxis::Avg, item_key, - Some(2), - Some(10), - Some(2), - Some(10), + Some(state(2, 10, 0x33)), + Some(state(2, 10, 0x33)), grove_version, ) .unwrap_add_cost(&mut noop_cost) @@ -3657,32 +4041,24 @@ mod bug2_avg_axis_mirror_tests { let entry = Element::get(&secondary, key.as_slice(), true, grove_version) .unwrap() .expect("entry present"); - assert_eq!(entry.sum_value_or_default(), 10, "payload must remain 10"); + assert_eq!( + entry.sum_value_or_default(), + 10, + "carried sum must remain 10" + ); } - /// A delete reaches the mirror as `new_count = new_sum = None`, which must - /// resolve to "no new key" and leave only the removal. The row has to - /// disappear from the axis secondary — not merely stop being findable — or - /// the secondary's own aggregate keeps counting it. + /// A delete reaches the mirror as a `None` new state, which must resolve + /// to "no new key" and leave only the removal. The row has to disappear + /// from the axis secondary — not merely stop being findable — or the + /// secondary's own aggregate keeps counting it. #[test] fn avg_axis_mirror_removes_the_row_when_the_new_state_is_absent() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); - let axes: Vec<(u8, Option>)> = vec![(IndexAxis::Avg.tag(), None)]; - db.insert( - [TEST_LEAF].as_ref(), - b"pcpsit", - Element::empty_provable_count_provable_sum_indexed_tree(axes).expect("canonical axes"), - None, - None, - grove_version, - ) - .unwrap() - .expect("empty PCPSIT insert"); + let (tx, batch) = open_axis_secondary(&db, IndexAxis::Avg, grove_version); - let tx = db.start_transaction(); let item_key = b"row"; - let batch = StorageBatch::new(); let path_segments: [&[u8]; 2] = [TEST_LEAF, b"pcpsit".as_ref()]; let path: SubtreePath<_> = (&path_segments).into(); let mut secondary = db @@ -3702,9 +4078,7 @@ mod bug2_avg_axis_mirror_tests { IndexAxis::Avg, item_key, None, - None, - Some(2), - Some(10), + Some(state(2, 10, 0x44)), grove_version, ) .unwrap() @@ -3730,9 +4104,7 @@ mod bug2_avg_axis_mirror_tests { &mut secondary, IndexAxis::Avg, item_key, - Some(2), - Some(10), - None, + Some(state(2, 10, 0x44)), None, grove_version, ) diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 586c55b27..1cfcf81ed 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -9,8 +9,13 @@ use std::collections::HashMap; -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; -use grovedb_merk::element::insert::ElementInsertToStorageExtensions; +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::element::{ + costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, + insert::ElementInsertToStorageExtensions, +}; use grovedb_merkle_mountain_range::{ hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, }; @@ -141,6 +146,38 @@ impl GroveDb { let updated_element = Element::new_mmr_tree(new_mmr_size, existing_flags); + // If the parent is an indexed primary, snapshot this entry's state + // BEFORE the rewrite. An append leaves `(count, sum)` alone but + // moves the entry's committed value hash, which is exactly what a + // canonical secondary row binds — so without this the row is left + // pointing at a commitment that no longer exists. The propagation + // walk cannot cover it: the mutation lands here, at the START path, + // rather than at a level the walk climbs through. + let parent_is_indexed_primary = parent_merk.tree_type.is_indexed_primary(); + let old_indexed_state = if parent_is_indexed_primary { + let old_value_hash = cost_return_on_error!( + &mut cost, + parent_merk + .get_value_hash( + key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let (count, sum) = element.count_sum_value_or_default(); + old_value_hash.map( + |value_hash| crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, + }, + ) + } else { + None + }; + // MMR root hash flows as the Merk child hash cost_return_on_error!( &mut cost, @@ -149,16 +186,79 @@ impl GroveDb { .map_err(|e| e.into()) ); - // 5. Propagate changes from parent upward + // 5. Mirror the moved entry into the primary's axis secondaries (if + // the parent is an indexed primary), then propagate upward, + // seeding the mirror's per-axis roots so the indexed element one + // level up is rebuilt over the NEW secondary state rather than + // re-read from stale root keys. + let initial_deferred = if parent_is_indexed_primary { + let (grandparent_path, indexed_key) = cost_return_on_error_no_add!( + cost, + path.derive_parent().ok_or(Error::CorruptedCodeExecution( + "an indexed primary requires a grandparent holding its indexed element", + )) + ); + let grandparent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + grandparent_path, + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + let indexed_element = cost_return_on_error!( + &mut cost, + Element::get(&grandparent_merk, indexed_key, true, grove_version) + .map_err(Error::MerkError) + ); + drop(grandparent_merk); + let is_multi_axis = matches!( + indexed_element.underlying(), + Element::ProvableCountProvableSumIndexedTree(..) + ); + let staged = cost_return_on_error!( + &mut cost, + self.mirror_indexed_primary_entry( + path.clone(), + &parent_merk, + &indexed_element, + key, + old_indexed_state, + tx.as_ref(), + &batch, + grove_version, + ) + ); + // The two seeds are not interchangeable: propagation rebuilds a + // single-axis element from `deferred_secondary` and a PCPSIT + // from `deferred_axes`, and seeding the wrong one leaves the + // other set for a later iteration that has no indexed element + // to apply it to. + if is_multi_axis { + (None, Some(staged)) + } else { + ( + staged.first().map(|(_, hash, key)| (*hash, key.clone())), + None, + ) + } + } else { + (None, None) + }; + let (initial_deferred_secondary, initial_deferred_axes) = initial_deferred; + let mut merk_cache: HashMap, Merk> = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction( + self.propagate_changes_with_transaction_with_initial_deferred_axes( merk_cache, path, + initial_deferred_secondary, + initial_deferred_axes, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 9d176506d..df9d1da3c 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -1840,7 +1840,7 @@ impl GroveDb { // multi-layer accounting (if any) reflects the consumed // slots. let limit_u64 = path_query.query.limit.map(|l| l as u64); - let prove_result = cost_return_on_error!( + let mut prove_result = cost_return_on_error!( &mut cost, subtree .prove_count_offset_on_range( @@ -1862,6 +1862,88 @@ impl GroveDb { e ))) ); + // Dereference reference rows before encoding. + // + // This short-circuit returns without reaching the main + // ref-rewriting loop below, which is why the count-offset flow + // used to reject reference entries outright. Running the same + // rewrite here closes that gap rather than bypassing it. + // + // These are ORDINARY user references, so they follow ordinary + // terminal-reference semantics — unlike an indexed secondary + // row, which binds its immediate primary node and is resolved + // by `indexed_axis::reference_resolution`. The two rules are + // deliberately separate code paths. + for op in prove_result.ops.iter_mut() { + let node = match op { + Op::Push(node) | Op::PushInverted(node) => node, + _ => continue, + }; + let Node::KVValueHashFeatureType(key, value, _, feature_type) = node else { + continue; + }; + let elem = match Element::deserialize(value, grove_version) { + Ok(e) => e.into_underlying(), + Err(_) => continue, + }; + let (Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..)) = elem + else { + continue; + }; + let absolute_path = match path_from_reference_path_type( + reference_path, + &path.to_vec(), + Some(key.as_slice()), + ) { + Ok(p) => p, + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + }; + let referenced_elem = cost_return_on_error!( + &mut cost, + self.follow_reference( + absolute_path.as_slice().into(), + true, + None, + grove_version + ) + ); + let serialized_referenced_elem = match referenced_elem.serialize(grove_version) { + Ok(bytes) => bytes, + Err(_) => { + return Err(Error::CorruptedData(String::from( + "unable to serialize element", + ))) + .wrap_with_cost(cost); + } + }; + let reference_element_hash = value_hash(value).unwrap_add_cost(&mut cost); + *node = match feature_type { + TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(count, sum) => { + Node::KVRefValueHashCountSum( + key.to_owned(), + serialized_referenced_elem, + reference_element_hash, + *count, + *sum, + ) + } + TreeFeatureType::ProvableCountedMerkNode(count) => Node::KVRefValueHashCount( + key.to_owned(), + serialized_referenced_elem, + reference_element_hash, + *count, + ), + other => { + return Err(Error::CorruptedData(format!( + "count-offset proof: reference row {} carries non-count feature type \ + {other:?}", + hex::encode(key) + ))) + .wrap_with_cost(cost); + } + }; + } let mut serialized = Vec::with_capacity(128); encode_into(prove_result.ops.iter(), &mut serialized); // Apply consumed limit slots to the outer accounting. diff --git a/grovedb/src/operations/proof/indexed_axis/generate.rs b/grovedb/src/operations/proof/indexed_axis/generate.rs index 432f8e849..5bb3d9e87 100644 --- a/grovedb/src/operations/proof/indexed_axis/generate.rs +++ b/grovedb/src/operations/proof/indexed_axis/generate.rs @@ -856,14 +856,32 @@ impl GroveDb { ); let descending = !secondary_query.left_to_right; let requested_limit = limit; - let sec_result = cost_return_on_error!( + let mut sec_result = cost_return_on_error!( &mut cost, secondary_merk - .prove(secondary_query, limit, grove_version) + .prove_without_encoding(secondary_query, limit, grove_version) .map_err(|e| Error::CorruptedData(format!( "indexed-axis range proof: secondary range proof: {e}" ))) ); + // Resolve canonical rows into the primary values they point at, + // before encoding, so the verified result carries the referenced + // Element rather than the reference. + cost_return_on_error!( + &mut cost, + super::reference_resolution::resolve_axis_reference_nodes( + sec_result.proof.iter_mut(), + axis, + &primary_merk, + self, + &path_keys, + transaction, + batch, + grove_version, + ) + ); + let mut serialized_secondary = Vec::with_capacity(128); + encode_into(sec_result.proof.iter(), &mut serialized_secondary); Ok(IndexedAxisRangeProof { axis_tag: axis.tag(), @@ -872,7 +890,7 @@ impl GroveDb { ancestor_attestations, other_axes_root_hashes, target_is_pcpsit, - secondary_proof: sec_result.proof, + secondary_proof: serialized_secondary, requested_limit, descending, }) @@ -993,7 +1011,7 @@ impl GroveDb { .wrap_with_cost(cost); } let inner_range = MerkQueryItemForRange::RangeFull(std::ops::RangeFull); - let prove_result = cost_return_on_error!( + let mut prove_result = cost_return_on_error!( &mut cost, secondary_merk .prove_count_offset_on_range( @@ -1007,6 +1025,23 @@ impl GroveDb { "indexed-axis paginated proof: secondary count-offset proof: {e}" ))) ); + // Resolve canonical rows into the primary values they point at, + // BEFORE encoding — the count-offset path used to return here + // without a reference post-pass, which is why it rejected + // reference rows outright. + cost_return_on_error!( + &mut cost, + super::reference_resolution::resolve_axis_reference_nodes( + prove_result.ops.iter_mut(), + axis, + &primary_merk, + self, + &path_keys, + transaction, + batch, + grove_version, + ) + ); let mut serialized = Vec::with_capacity(128); encode_into(prove_result.ops.iter(), &mut serialized); @@ -1263,6 +1298,11 @@ impl GroveDb { cost, build_paginated_secondary_proof( &secondary_merk, + &primary_merk, + self, + &path_keys, + transaction, + batch, *offset, *k, axis_query.descending, @@ -1280,6 +1320,11 @@ impl GroveDb { cost, build_paginated_secondary_proof( &secondary_merk, + &primary_merk, + self, + &path_keys, + transaction, + batch, rank_offset, 1, axis_query.descending, @@ -1304,15 +1349,30 @@ impl GroveDb { cost, crate::query::axis_lowering::axis_bounded_merk_query(axis_query) ); - let sec_result = cost_return_on_error!( + let mut sec_result = cost_return_on_error!( &mut cost, secondary_merk - .prove(secondary_query, Some(*limit), grove_version) + .prove_without_encoding(secondary_query, Some(*limit), grove_version) .map_err(|e| Error::CorruptedData(format!( "axis descent: secondary range proof: {e}" ))) ); - sec_result.proof + cost_return_on_error!( + &mut cost, + super::reference_resolution::resolve_axis_reference_nodes( + sec_result.proof.iter_mut(), + axis, + &primary_merk, + self, + &path_keys, + transaction, + batch, + grove_version, + ) + ); + let mut serialized = Vec::with_capacity(128); + encode_into(sec_result.proof.iter(), &mut serialized); + serialized } } AxisTraversal::AggregateOverValueRange { lo, hi, fold } => { @@ -1349,8 +1409,14 @@ impl GroveDb { /// The count-offset paginated secondary proof shared by the `TopK` and /// `RankOfKey` embedded traversals — the same shape step 4 of /// `build_indexed_axis_paginated_proof` emits. +#[allow(clippy::too_many_arguments)] fn build_paginated_secondary_proof<'db, S>( secondary_merk: &grovedb_merk::Merk, + primary_merk: &grovedb_merk::Merk, + grovedb: &'db GroveDb, + primary_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, offset: u64, k: u16, descending: bool, @@ -1369,7 +1435,7 @@ where ))); } let inner_range = MerkQueryItemForRange::RangeFull(std::ops::RangeFull); - let prove_result = secondary_merk + let mut prove_result = secondary_merk .prove_count_offset_on_range( &inner_range, offset, @@ -1381,6 +1447,19 @@ where .map_err(|e| { Error::CorruptedData(format!("axis descent: secondary count-offset proof: {e}")) })?; + // Resolve canonical rows into the primary values they point at, + // before encoding. + super::reference_resolution::resolve_axis_reference_nodes( + prove_result.ops.iter_mut(), + axis, + primary_merk, + grovedb, + primary_path, + transaction, + batch, + grove_version, + ) + .unwrap_add_cost(cost)?; let mut serialized = Vec::with_capacity(128); encode_into(prove_result.ops.iter(), &mut serialized); Ok(serialized) diff --git a/grovedb/src/operations/proof/indexed_axis/mod.rs b/grovedb/src/operations/proof/indexed_axis/mod.rs index 7508c14f7..15f846b50 100644 --- a/grovedb/src/operations/proof/indexed_axis/mod.rs +++ b/grovedb/src/operations/proof/indexed_axis/mod.rs @@ -65,6 +65,7 @@ mod axis_api; mod envelope; #[cfg(feature = "minimal")] mod generate; +pub(crate) mod reference_resolution; pub(crate) mod verify; pub use envelope::{ diff --git a/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs b/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs new file mode 100644 index 000000000..dc70b3553 --- /dev/null +++ b/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs @@ -0,0 +1,362 @@ +//! Resolving canonical secondary rows into proof nodes that carry the +//! primary value they point at. +//! +//! Every indexed secondary row is a canonical one-hop +//! `ReferenceWithSumItem(SiblingReference(primary_key), Some(1), sum)` +//! (see [`axis_row_reference`]). Merk emits such a row as a +//! `KVValueHashFeatureType` node carrying the REFERENCE's bytes, and +//! leaves dereferencing to the layer above — the same division of labour +//! the regular count-tree proof flow uses. This module is that layer for +//! indexed axes. +//! +//! # The logical-origin rule +//! +//! A secondary Merk lives at a derived storage prefix +//! (`blake3(primary_prefix ‖ axis_tag)`) and has **no `SubtreePath`** — +//! there is no GroveDB path that names it, so the generic path-keyed +//! reference machinery (`follow_reference` / `MerkCache`) cannot express +//! a row's origin at all. Resolution here is therefore purpose-built: +//! a row's logical origin is the INDEXED PRIMARY's path, so +//! `SiblingReference(primary_key)` resolves to `primary_key` in the +//! primary Merk. Nothing in this module treats the derived secondary +//! prefix as a user-visible path, and no caller may. +//! +//! # What the rewrite binds +//! +//! The emitted node is `KVRefValueHash{Count,CountSum}(key, +//! target_bytes, H(reference bytes), ..)`, whose hash reconstruction is +//! `combine_hash(H(reference bytes), H(target_bytes))`. That is exactly +//! the row's committed value hash, so the resolved target value is bound +//! into the secondary root — which the axis verifier in turn binds to the +//! indexed element via `combine_hash_three`. Substituting either the +//! target value or the reference breaks the root. +//! +//! # Immediate-node binding +//! +//! Resolution stops at the primary entry. It does NOT follow a chain to a +//! terminal, because a canonical row binds the IMMEDIATE primary node's +//! committed value hash — see [`INDEXED_SECONDARY_MAX_HOP`]. Ordinary +//! user references keep their terminal semantics; this one-hop rule is +//! selected explicitly here and applies only to indexed rows. +//! +//! [`axis_row_reference`]: crate::operations::indexed_tree::axis_row_reference +//! [`INDEXED_SECONDARY_MAX_HOP`]: +//! crate::operations::indexed_tree::INDEXED_SECONDARY_MAX_HOP + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; +use grovedb_element::indexed::IndexAxis; +use grovedb_merk::{ + element::{costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions}, + proofs::{Node, Op}, + tree::{combine_hash, value_hash}, + Merk, TreeFeatureType, +}; +use grovedb_storage::{StorageBatch, StorageContext}; +use grovedb_version::version::GroveVersion; + +use crate::{ + operations::indexed_tree::{axis_sort_key_len, indexed_row_target_key}, + Element, Error, GroveDb, Transaction, +}; + +/// Rewrite every canonical reference row in a secondary proof's op stream +/// into a resolved-value node. +/// +/// `primary_merk` must be the indexed primary the secondary mirrors — +/// that is the row's logical origin, and reading the target anywhere else +/// would resolve a `SiblingReference` against the wrong parent. +/// +/// Rows that are not references are left untouched, so this is safe to +/// run over any secondary proof. +#[allow(clippy::too_many_arguments)] +pub(crate) fn resolve_axis_reference_nodes<'a, 'db, S: StorageContext<'db>>( + ops: impl IntoIterator, + axis: IndexAxis, + primary_merk: &Merk, + grovedb: &'db GroveDb, + primary_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult<(), Error> { + let mut cost = OperationCost::default(); + let sort_len = axis_sort_key_len(axis); + + for op in ops { + let node = match op { + Op::Push(node) | Op::PushInverted(node) => node, + _ => continue, + }; + let Node::KVValueHashFeatureType(key, value, _committed_value_hash, feature_type) = node + else { + continue; + }; + + // Only canonical rows are rewritten. Anything else in a secondary + // is corruption, but this pass is not the checker for that — the + // node is left alone and the shape checks reject it downstream. + let Ok(row) = Element::deserialize(value.as_slice(), grove_version) else { + continue; + }; + if !row.is_reference() { + continue; + } + if key.len() < sort_len { + return Err(Error::CorruptedData(format!( + "indexed-axis reference resolution: secondary key {} is shorter than the \ + {axis:?} axis sort key ({sort_len} bytes)", + hex::encode(key) + ))) + .wrap_with_cost(cost); + } + let key_suffix = key[sort_len..].to_vec(); + // Enforces the canonical shape AND that the reference target + // agrees with the secondary-key suffix. Both are needed: the + // suffix is what the verifier decodes the primary key from, so a + // row whose commitment points elsewhere would authenticate a + // value filed under a key it does not belong to. + let (target_key, _carried_sum) = cost_return_on_error!( + &mut cost, + indexed_row_target_key(&row, &key_suffix, "indexed-axis reference resolution") + .wrap_with_cost(OperationCost::default()) + ); + let target_key = target_key.to_vec(); + + let (target_element, target_value_hash) = cost_return_on_error!( + &mut cost, + Element::get_with_value_hash(primary_merk, &target_key, true, grove_version).map_err( + |e| Error::CorruptedData(format!( + "indexed-axis reference resolution: reading primary entry {}: {e}", + hex::encode(&target_key) + )) + ) + ); + + let target_bytes = cost_return_on_error!( + &mut cost, + target_element + .serialize(grove_version) + .map_err(|e| Error::CorruptedData(format!( + "indexed-axis reference resolution: serializing primary entry {}: {e}", + hex::encode(&target_key) + ))) + .wrap_with_cost(OperationCost::default()) + ); + + // The reference element's OWN value hash — not the node's stored + // (combined) hash. This is the half of the commitment the node + // carries explicitly; the other half the verifier recomputes. + let reference_element_hash = value_hash(value.as_slice()).unwrap_add_cost(&mut cost); + + // Which node family can express this target's commitment? + // + // `KVRefValueHashCountSum` reconstructs the row's hash as + // `combine_hash(H(reference), H(target_bytes))`, which is correct + // only when the target's own committed hash IS `H(target_bytes)` — + // true for item-shaped entries. A tree-shaped entry (the NORMAL + // case under a count-indexed primary, whose whole purpose is + // indexing its children's counts) commits a combined hash instead, + // and no choice of target bytes makes `H(bytes)` equal it. Those + // get the target-witness variant, which carries the missing + // child commitment so the verifier can rebuild both combines. + let simple_target_hash = value_hash(&target_bytes).unwrap_add_cost(&mut cost); + let target_child_hash = if simple_target_hash == target_value_hash { + None + } else { + // Recover the child commitment the target folded in. It is + // read from the target's own subtree root rather than derived + // from the committed hash (which is one-way), and then checked + // against that committed hash below — so a wrong witness is + // caught here rather than becoming an unverifiable proof. + let child_hash = cost_return_on_error!( + &mut cost, + target_layered_child_hash( + &target_element, + primary_path, + &target_key, + grovedb, + transaction, + batch, + grove_version, + ) + ); + let rebuilt = combine_hash(&simple_target_hash, &child_hash).unwrap_add_cost(&mut cost); + if rebuilt != target_value_hash { + return Err(Error::NotSupported(format!( + "indexed-axis proofs cannot yet witness the committed value hash of primary \ + entry {} ({}): its stored hash is a combined hash that \ + combine_hash(H(value), child_root) does not reproduce, so no single \ + child-hash witness expresses it. Reads, writes and integrity verification \ + handle this entry normally; only proving it is unsupported.", + hex::encode(&target_key), + target_element.type_str() + ))) + .wrap_with_cost(cost); + } + Some(child_hash) + }; + + *node = match (feature_type, target_child_hash) { + ( + TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(count, sum), + Some(child_hash), + ) => Node::KVRefValueHashCountSumWithTargetChildHash( + key.clone(), + target_bytes, + reference_element_hash, + *count, + *sum, + child_hash, + ), + (TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(count, sum), None) => { + Node::KVRefValueHashCountSum( + key.clone(), + target_bytes, + reference_element_hash, + *count, + *sum, + ) + } + (TreeFeatureType::ProvableCountedMerkNode(count), None) => { + Node::KVRefValueHashCount(key.clone(), target_bytes, reference_element_hash, *count) + } + (TreeFeatureType::ProvableCountedMerkNode(_), Some(_)) => { + return Err(Error::NotSupported( + "indexed-axis proofs do not support a layered target under a count-only \ + secondary; every axis secondary is a dual-aggregate \ + ProvableCountProvableSumTree, so this shape should be unreachable" + .to_string(), + )) + .wrap_with_cost(cost); + } + (other, _) => { + return Err(Error::CorruptedData(format!( + "indexed-axis reference resolution: secondary row {} carries feature type \ + {other:?}; every axis secondary is a dual-aggregate \ + ProvableCountProvableSumTree, so its rows must be count-bearing", + hex::encode(key) + ))) + .wrap_with_cost(cost); + } + }; + } + + Ok(()).wrap_with_cost(cost) +} + +/// The child commitment a LAYERED primary entry folded into its stored +/// value hash — the second half of +/// `combine_hash(H(element bytes), child)`. +/// +/// Two shapes reach here: +/// +/// - **Subtree-ish elements** (`Tree`, `SumTree`, the `Provable*` family, +/// and the non-Merk trees): the child is the subtree's root hash, read +/// by opening the subtree. +/// - **Reference-shaped entries**: the child is the NEXT target's +/// committed value hash, read one hop along. +/// +/// An indexed-tree element commits with `combine_hash_three` (value, +/// primary root, axes digest) and therefore has no single child witness; +/// it is rejected by the caller's rebuild check rather than here, so the +/// rejection is driven by the hash not reconstructing rather than by a +/// type list that could drift. +#[allow(clippy::too_many_arguments)] +fn target_layered_child_hash<'db>( + target_element: &Element, + primary_path: &[Vec], + target_key: &[u8], + grovedb: &'db GroveDb, + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult { + let mut cost = OperationCost::default(); + + if let Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) = target_element + { + // One hop only: the primary entry's own commitment folds in its + // IMMEDIATE target's hash, so that is the witness — following + // further would bind something the primary node does not. + let mut qualified = primary_path.to_vec(); + qualified.push(target_key.to_vec()); + let next_absolute = match crate::reference_path::path_from_reference_path_type( + reference_path.clone(), + &qualified, + Some(target_key), + ) { + Ok(p) => p, + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + }; + let Some((next_key, next_parent)) = next_absolute.split_last() else { + return Err(Error::CorruptedData( + "indexed-axis reference resolution: a reference-shaped primary entry resolves \ + to an empty path" + .to_string(), + )) + .wrap_with_cost(cost); + }; + let next_parent_refs: Vec<&[u8]> = next_parent.iter().map(|s| s.as_slice()).collect(); + let next_merk = cost_return_on_error!( + &mut cost, + grovedb.open_transactional_merk_at_path( + next_parent_refs.as_slice().into(), + transaction, + Some(batch), + grove_version, + ) + ); + let next_value_hash = cost_return_on_error!( + &mut cost, + next_merk + .get_value_hash( + next_key.as_slice(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|e| Error::CorruptedData(format!( + "indexed-axis reference resolution: reading the next hop's value hash: {e}" + ))) + ); + let resolved = match next_value_hash { + Some(h) => h, + None => { + return Err(Error::CorruptedReferencePathKeyNotFound(format!( + "indexed-axis reference resolution: a reference-shaped primary entry points \ + at {}, which does not exist", + hex::encode(next_key) + ))) + .wrap_with_cost(cost); + } + }; + return Ok(resolved).wrap_with_cost(cost); + } + + // Subtree-ish: the child is the subtree's own root hash. + let mut subtree_path = primary_path.to_vec(); + subtree_path.push(target_key.to_vec()); + let subtree_refs: Vec<&[u8]> = subtree_path.iter().map(|s| s.as_slice()).collect(); + let subtree = cost_return_on_error!( + &mut cost, + grovedb.open_transactional_merk_at_path( + subtree_refs.as_slice().into(), + transaction, + Some(batch), + grove_version, + ) + ); + let (root_hash, ..) = cost_return_on_error!( + &mut cost, + subtree + .root_hash_key_and_aggregate_data() + .map_err(|e| Error::CorruptedData(format!( + "indexed-axis reference resolution: reading a layered primary entry's subtree \ + root: {e}" + ))) + ); + Ok(root_hash).wrap_with_cost(cost) +} diff --git a/grovedb/src/operations/proof/mod.rs b/grovedb/src/operations/proof/mod.rs index 0f3aeec78..afc6f2b47 100644 --- a/grovedb/src/operations/proof/mod.rs +++ b/grovedb/src/operations/proof/mod.rs @@ -989,6 +989,23 @@ fn node_to_string(node: &Node) -> Result { hex::encode(value_hash), count ), + Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + count, + sum, + target_child_hash, + ) => format!( + "KVRefValueHashCountSumWithTargetChildHash({}, {}, HASH[{}], {}, {}, \ + target_child=HASH[{}])", + hex_to_ascii(key), + element_hex_to_ascii(value)?, + hex::encode(value_hash), + count, + sum, + hex::encode(target_child_hash) + ), Node::KVRefValueHash(key, value, value_hash) => format!( "KVRefValueHash({}, {}, HASH[{}])", hex_to_ascii(key), diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 7510b3cdf..816abf7d1 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -629,11 +629,10 @@ impl GroveDb { // normal traversal (own_count = 0) and not surfaced via // the merk's `returned_items`. If one appears here, the // proof was forged. - // • **Reference / ReferenceWithSumItem** — would need the - // regular flow's reference post-pass to dereference the - // target; we don't run that on the count-offset - // short-circuit, so a raw reference here would be returned - // verbatim. Reject. + // • **Raw Reference / ReferenceWithSumItem** — the prover now + // runs a reference post-pass on this short-circuit, so an + // honest proof surfaces the dereferenced TARGET here, never + // the reference itself. An unresolved one is malformed. // • **Non-empty tree** — V1 strict-mode would require a // `KVValueHashFeatureTypeWithChildHash` proof node here; // accepting one without that would silently bypass the @@ -683,16 +682,23 @@ impl GroveDb { hex::encode(&item.key) ))); } + // A RAW reference must not reach the caller: the prover's + // post-pass rewrites reference rows into resolved-value nodes, + // so by this point `item.value` is the dereferenced TARGET and + // `item.reference_element_hash` records that a reference was + // resolved. Seeing an unresolved reference here means the proof + // skipped the post-pass — reject rather than surface it. if inner.is_reference() { - return Err(Error::NotSupported(format!( - "count-offset paginated proofs do not yet support \ - Reference / ReferenceWithSumItem return values (key {}); the \ - regular flow's reference post-pass isn't applied on the \ - count-offset short-circuit, so an accepted reference here \ - would surface the raw Element::Reference rather than the \ - dereferenced target", - hex::encode(&item.key) - ))); + return Err(Error::InvalidProof( + query.clone(), + format!( + "count-offset paginated proof surfaced a raw \ + Reference / ReferenceWithSumItem at key {} — the prover's \ + reference post-pass rewrites these into resolved-value \ + nodes, so an unresolved reference here is malformed", + hex::encode(&item.key) + ), + )); } // Empty-tree value-hash equality check (defense-in-depth on // top of the merk-level KV→KVValueHash forgery guard). @@ -4234,7 +4240,10 @@ impl GroveDb { | Node::KVSum(key, value, ..) | Node::KVRefValueHashSum(key, value, ..) | Node::KVCountSum(key, value, ..) - | Node::KVRefValueHashCountSum(key, value, ..) => Some((key.clone(), value.clone())), + | Node::KVRefValueHashCountSum(key, value, ..) + | Node::KVRefValueHashCountSumWithTargetChildHash(key, value, ..) => { + Some((key.clone(), value.clone())) + } // These nodes don't have values, only key+hash or just hash Node::KVDigest(..) | Node::KVDigestCount(..) diff --git a/grovedb/src/tests/count_offset_paginated_tests.rs b/grovedb/src/tests/count_offset_paginated_tests.rs index 964e0f66b..f0602a6c3 100644 --- a/grovedb/src/tests/count_offset_paginated_tests.rs +++ b/grovedb/src/tests/count_offset_paginated_tests.rs @@ -841,13 +841,16 @@ mod tests { // `merk/src/proofs/query/count_offset/tests.rs`) covers the // verifier symmetric. - /// Prover-side rejection for `Reference` in-range entries. Earlier - /// drafts returned the raw `Element::Reference` bytes verbatim - /// because the count-offset short-circuit doesn't run the regular - /// flow's reference post-pass. The prover now refuses to emit - /// these. + /// `Reference` in-range entries are RESOLVED, not rejected. + /// + /// The count-offset short-circuit used to return before the regular + /// flow's reference post-pass, so the prover refused to emit reference + /// rows at all rather than surface raw `Element::Reference` bytes. The + /// short-circuit now runs the post-pass itself, so a verified result + /// carries the dereferenced TARGET — which is what the regular flow + /// has always returned for the same query. #[test] - fn rejects_count_offset_with_reference_entry() { + fn count_offset_resolves_reference_entries_to_their_target() { let v = GroveVersion::latest(); let db = make_test_grovedb(v); db.insert( @@ -903,14 +906,36 @@ mod tests { vec![b"counts".to_vec()], SizedQuery::new(q, Some(2), Some(1)), ); - let result = db.prove_query(&path_query, None, v).unwrap(); - let err = result.expect_err("prover must reject Reference in-range entry"); - let msg = format!("{}", err); - assert!( - msg.contains("Reference"), - "prover rejection should mention Reference; got {}", - msg + let proof = db + .prove_query(&path_query, None, v) + .unwrap() + .expect("a Reference in-range entry must now be provable"); + let (root_hash, verified) = + GroveDb::verify_query(&proof, &path_query, v).expect("proof must verify"); + assert_eq!( + root_hash, + db.root_hash(None, v).unwrap().unwrap(), + "the resolved proof must still reconstruct the grove root" + ); + + // offset 1 skips "a"; the page is ["b" (the reference), "c"]. + let values: Vec<(Vec, Element)> = verified + .into_iter() + .map(|(_, key, element)| (key, element.expect("value present"))) + .collect(); + assert_eq!( + values.len(), + 2, + "limit 2 after offset 1 must return two rows, got {values:?}" ); + assert_eq!(values[0].0, b"b".to_vec()); + assert_eq!( + values[0].1, + Element::new_item(b"target_value".to_vec()), + "the reference row must surface its dereferenced TARGET, not the reference" + ); + assert_eq!(values[1].0, b"c".to_vec()); + assert_eq!(values[1].1, Element::new_item(b"v_c".to_vec())); } // ──────── check_count_offset_target_tree_type error normalization ──────── @@ -1246,11 +1271,16 @@ mod tests { ); } - /// Defense-in-depth: a forged proof that surfaces a Reference - /// element in `returned_items` must be rejected as `NotSupported` - /// mentioning "Reference" — the count-offset short-circuit doesn't - /// run the regular flow's reference post-pass, so accepting one - /// would surface a raw `Element::Reference` to the caller. + /// Defense-in-depth: a forged proof that surfaces an UNRESOLVED + /// Reference in `returned_items` must be rejected as `InvalidProof`. + /// + /// The prover now runs a reference post-pass on the count-offset + /// short-circuit, so an honest proof surfaces the dereferenced TARGET + /// and never the reference itself (see + /// `count_offset_resolves_reference_entries_to_their_target`). A raw + /// reference reaching the caller therefore means the value was + /// substituted after the fact, not that the shape is unsupported — + /// hence `InvalidProof` rather than `NotSupported`. #[test] fn verifier_rejects_forged_reference_returned_item() { use crate::reference_path::ReferencePathType; @@ -1265,8 +1295,8 @@ mod tests { let result = GroveDb::verify_query_raw(&tampered, &path_query, v); let err = result.expect_err("forged Reference return must be rejected"); assert!( - matches!(err, crate::Error::NotSupported(ref msg) if msg.contains("Reference")), - "forged Reference return should reject as NotSupported mentioning Reference; got {:?}", + matches!(err, crate::Error::InvalidProof(_, ref msg) if msg.contains("Reference")), + "forged Reference return should reject as InvalidProof mentioning Reference; got {:?}", err, ); } diff --git a/grovedb/src/tests/coverage_misc_tests.rs b/grovedb/src/tests/coverage_misc_tests.rs index acc99ec15..dafc90341 100644 --- a/grovedb/src/tests/coverage_misc_tests.rs +++ b/grovedb/src/tests/coverage_misc_tests.rs @@ -412,20 +412,24 @@ mod tests { ) .storage_cost .added_bytes; - // +8 bytes of primary key => +8 bytes in each of the two rows the - // mirror writes. - assert_eq!(k16_count - k8_count, 16); - // A 8-byte primary + the avg axis and a 16-byte primary + the - // count axis both produce a 24-byte secondary key, and since - // issue #806 EVERY axis is a dual-aggregate - // ProvableCountProvableSumTree (the count axis mirrors its - // count_value into the sum half), the merk-node aggregate width - // is identical across axes. What is left is only the payload - // shape: avg stores an `ItemWithSumItem` (12 bytes at worst sum - // width) where count stores a `SumItem` (11) — the mirror - // estimator sizes each axis's row with its REAL payload shape. - assert_eq!(k8_avg - k16_count, 1); - assert_eq!(k8_avg - k8_count, 16 + 1); + // +8 bytes of primary key costs 24, not 16, because a canonical + // row carries the primary key TWICE: once in the secondary key + // (both the delete and the insert widen, +8 each) and once inside + // the row's `SiblingReference` (only the insert adds bytes, +8). + assert_eq!(k16_count - k8_count, 8 + 8 + 8); + // Every axis now writes the SAME row shape — a canonical + // `ReferenceWithSumItem` — so an axis comparison isolates two + // things only: the sort-key width, and the length of the primary + // key the row references. + // + // 8-byte primary + avg (16-byte sort) and 16-byte primary + count + // (8-byte sort) produce the same 24-byte secondary key, so what is + // left is the referenced key: avg's rows reference an 8-byte key + // where count's reference a 16-byte one. + assert_eq!(k16_count - k8_avg, 8); + // Same primary key, wider sort key: +8 bytes on each of the two + // rows the mirror writes, and no change to the referenced key. + assert_eq!(k8_avg - k8_count, 16); let count_axis = narrow; let sizes = EstimatedLayerSizes::AllItems(8, 100, None); @@ -448,10 +452,26 @@ mod tests { #[test] fn indexed_secondary_mirror_key_width_saturates_at_u8_max() { let axes = [IndexAxis::Avg]; + let at_250 = mirror_cost(EstimatedLayerSizes::AllItems(250, 100, None), &axes); + let at_255 = mirror_cost(EstimatedLayerSizes::AllItems(255, 100, None), &axes); + + // The secondary KEY still saturates: 250+16 and 255+16 both clamp + // to 255 bytes, so the tree-shape work the key drives is identical. assert_eq!( - mirror_cost(EstimatedLayerSizes::AllItems(250, 100, None), &axes), - mirror_cost(EstimatedLayerSizes::AllItems(255, 100, None), &axes), - "both 250+16 and 255+16 clamp to a 255-byte secondary key" + (at_250.seek_count, at_250.hash_node_calls), + (at_255.seek_count, at_255.hash_node_calls), + "a clamped secondary key must do identical seek/hash work" + ); + + // The row VALUE does not saturate, and must not: a canonical row + // carries the primary key inside its `SiblingReference`, so a + // longer primary key means a genuinely larger row even once the + // secondary key has clamped. An estimate that saturated here would + // under-charge `added_bytes`, which is the one dimension a storage + // fee reservation cannot come in under. + assert!( + at_255.storage_cost.added_bytes > at_250.storage_cost.added_bytes, + "a longer referenced primary key must cost more even when the secondary key clamps: {at_250:?} vs {at_255:?}" ); } } diff --git a/grovedb/src/tests/provable_count_sum_tree_tests.rs b/grovedb/src/tests/provable_count_sum_tree_tests.rs index ac862bf8c..986a64849 100644 --- a/grovedb/src/tests/provable_count_sum_tree_tests.rs +++ b/grovedb/src/tests/provable_count_sum_tree_tests.rs @@ -103,6 +103,7 @@ mod tests { Node::KVCountSum(k, ..) => k.clone(), Node::KVDigestCountSum(k, ..) => k.clone(), Node::KVRefValueHashCountSum(k, ..) => k.clone(), + Node::KVRefValueHashCountSumWithTargetChildHash(k, ..) => k.clone(), Node::KVHashCount(..) => vec![], Node::Hash(_) | Node::KVHash(_) => vec![], // HashWithCount is keyless (collapsed subtree representation diff --git a/merk/src/merk/chunks.rs b/merk/src/merk/chunks.rs index 94fdb30fa..bc6d23dd7 100644 --- a/merk/src/merk/chunks.rs +++ b/merk/src/merk/chunks.rs @@ -487,6 +487,9 @@ mod test { Node::KVCount(..) => counts.kv += 1, Node::KVHashCount(..) => counts.kv_hash += 1, Node::KVRefValueHashCount(..) => counts.kv_ref_value_hash += 1, + Node::KVRefValueHashCountSumWithTargetChildHash(..) => { + counts.kv_ref_value_hash += 1 + } // HashWithCount is hash-equivalent to Hash for the verifier; // count it under `hash` for the test counter. Node::HashWithCount(..) => counts.hash += 1, diff --git a/merk/src/proofs/branch/mod.rs b/merk/src/proofs/branch/mod.rs index 91e318d8d..47ef06a0a 100644 --- a/merk/src/proofs/branch/mod.rs +++ b/merk/src/proofs/branch/mod.rs @@ -125,7 +125,8 @@ impl TrunkQueryResult { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) => Some(key.clone()), + | Node::KVRefValueHashCountSum(key, ..) + | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.clone()), Node::Hash(_) | Node::KVHash(_) | Node::KVHashCount(..) @@ -401,7 +402,8 @@ impl BranchQueryResult { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) => Some(key.clone()), + | Node::KVRefValueHashCountSum(key, ..) + | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.clone()), Node::Hash(_) | Node::KVHash(_) | Node::KVHashCount(..) diff --git a/merk/src/proofs/query/count_offset/emit.rs b/merk/src/proofs/query/count_offset/emit.rs index 392e2245c..ed2ac2245 100644 --- a/merk/src/proofs/query/count_offset/emit.rs +++ b/merk/src/proofs/query/count_offset/emit.rs @@ -258,12 +258,6 @@ where // Silently dropping it would be a correctness divergence — we // reject upfront instead. // - // • **Reference / ReferenceWithSumItem** in-range entry: regular - // GroveDB's reference post-pass dereferences these into the - // target's value bytes. The count-offset short-circuit returns - // before that post-pass, so a verified result would contain - // the raw `Element::Reference` rather than the target. Reject. - // // • **Non-empty tree** in-range entry: V1 strict-mode requires a // `KVValueHashFeatureTypeWithChildHash` proof node for these, // which the count-offset prover doesn't emit. The verifier @@ -271,8 +265,21 @@ where // time saves the work of producing an honest-but-unverifiable // proof. // - // Lifting any of these is straightforward future work: emit the - // appropriate node variant and update the verifier symmetrically. + // **References are supported.** They emit as + // `KVValueHashFeatureType` (see `emit_returned_node`), which the + // callers' reference post-pass rewrites into the + // `KVRefValueHash{Count,CountSum}` family carrying the resolved + // target — the same rewrite the regular count-tree flow performs. + // The verifier accepts and authenticates those variants, so a + // verified result carries the dereferenced target rather than a raw + // `Element::Reference`. A caller that does NOT run the post-pass + // still gets a sound proof; it just surfaces the reference bytes, + // which is why GroveDB's indexed-axis path applies the post-pass + // unconditionally. + // + // Lifting the remaining rejection is straightforward future work: + // emit the appropriate node variant and update the verifier + // symmetrically. if is_in_range { if own_struct == 0 { return Err(Error::InvalidProofError( @@ -288,17 +295,6 @@ where match Element::deserialize(value_bytes, grove_version) { Ok(elem) => { let inner = elem.into_underlying(); - if inner.is_reference() { - return Err(Error::InvalidProofError( - "count-offset paginated proofs do not yet support \ - Reference / ReferenceWithSumItem in-range entries — the regular \ - flow's reference post-pass isn't applied on the count-offset \ - short-circuit, so a verified result would expose the raw \ - Element::Reference rather than the dereferenced target" - .to_string(), - )) - .wrap_with_cost(cost); - } if inner.is_non_empty_tree() { return Err(Error::InvalidProofError( "count-offset paginated proofs do not yet support non-empty tree \ diff --git a/merk/src/proofs/query/count_offset/tests.rs b/merk/src/proofs/query/count_offset/tests.rs index bc7f9a37a..a8eaf8e60 100644 --- a/merk/src/proofs/query/count_offset/tests.rs +++ b/merk/src/proofs/query/count_offset/tests.rs @@ -185,6 +185,8 @@ fn returned_items_carry_full_committed_payload() { value: vec![5u8], value_hash: crate::tree::value_hash(&[5u8]).unwrap(), child_hash_verified: false, + // A directly-valued row, not a resolved reference. + reference_element_hash: None, }; assert_eq!( verified.returned_items[0], expected_f, diff --git a/merk/src/proofs/query/count_offset/verify.rs b/merk/src/proofs/query/count_offset/verify.rs index 0870242d0..727fa1587 100644 --- a/merk/src/proofs/query/count_offset/verify.rs +++ b/merk/src/proofs/query/count_offset/verify.rs @@ -73,9 +73,12 @@ pub struct CountOffsetReturnedItem { /// The matched key. pub key: Vec, /// The element's serialized value bytes, as emitted by the prover. - /// GroveDB's reference-resolution post-pass (mirroring the regular - /// count-tree proof flow) operates on this byte stream — reference - /// dereferencing happens at the GroveDB layer, not here. + /// + /// For a `KVRefValueHash*` node these are the RESOLVED target's + /// bytes, not the reference's — GroveDB's reference post-pass runs + /// before the proof is encoded, so by verify time the dereferencing + /// has already happened and `reference_element_hash` is what records + /// that this row was a reference. pub value: Vec, /// The value-hash the proof's merk node committed for this entry. /// For `KVCount` nodes this is `H(value)` (the Item-flavored value @@ -100,6 +103,19 @@ pub struct CountOffsetReturnedItem { /// `child_hash_verified = true` for non-empty trees); callers must /// not silently treat a `false` here as `true`. pub child_hash_verified: bool, + /// For a resolved-reference row (`KVRefValueHash*`), the value hash + /// of the REFERENCE element itself — the first half of the row's + /// committed `combine_hash(reference_element_hash, H(value))`. + /// `None` for a directly-valued row. + /// + /// This is what lets a caller authenticate the reference's own + /// content without the proof carrying its bytes: a caller that knows + /// the canonical shape a row must have can reconstruct those bytes + /// and check `value_hash(reconstructed) == reference_element_hash`. + /// GroveDB's indexed-axis verifier does exactly that, which is how a + /// row's `SiblingReference(primary_key)` / hop budget / carried sum + /// get authenticated rather than assumed. + pub reference_element_hash: Option, } /// The verifier's reconstructed view of an offset-paginated count-tree @@ -173,7 +189,14 @@ pub fn verify_count_offset_on_range_proof( | Node::KVValueHashFeatureType(_, _, _, _) | Node::HashWithCountAndSum(_, _, _, _, _) | Node::KVDigestCountSum(_, _, _, _) - | Node::KVCountSum(_, _, _, _) => Ok(()), + | Node::KVCountSum(_, _, _, _) + // Resolved-reference returns. GroveDB's post-pass rewrites a + // reference row's `KVValueHashFeatureType` into these before + // encoding, so the value bytes are the dereferenced target's + // and the node's own hash field is the reference element's. + | Node::KVRefValueHashCount(_, _, _, _) + | Node::KVRefValueHashCountSum(_, _, _, _, _) + | Node::KVRefValueHashCountSumWithTargetChildHash(_, _, _, _, _, _) => Ok(()), other => Err(Error::InvalidProofError(format!( "unexpected node type in count-offset proof: {}", other @@ -245,6 +268,11 @@ fn aggregate_of_proof_tree_node(tree: &ProofTree) -> Result { Node::HashWithCountAndSum(_, _, _, c, _) => Ok(*c), Node::KVDigestCountSum(_, _, c, _) => Ok(*c), Node::KVCountSum(_, _, c, _) => Ok(*c), + // Resolved-reference returns carry their aggregates in the same + // conceptual position as the value-bearing variants. + Node::KVRefValueHashCount(_, _, _, c) => Ok(*c), + Node::KVRefValueHashCountSum(_, _, _, c, _) => Ok(*c), + Node::KVRefValueHashCountSumWithTargetChildHash(_, _, _, c, _, _) => Ok(*c), Node::KVValueHashFeatureType(_, _, _, ft) => match ft { TreeFeatureType::ProvableCountedMerkNode(c) => Ok(*c), TreeFeatureType::ProvableCountedSummedMerkNode(c, _) => Ok(*c), @@ -399,6 +427,10 @@ fn verify_count_offset_shape( // Dual-axis (PCPS) per-element variants. Node::KVDigestCountSum(key, _, _, _) => key.as_slice(), Node::KVCountSum(key, _, _, _) => key.as_slice(), + // Resolved-reference per-element variants. + Node::KVRefValueHashCount(key, _, _, _) => key.as_slice(), + Node::KVRefValueHashCountSum(key, _, _, _, _) => key.as_slice(), + Node::KVRefValueHashCountSumWithTargetChildHash(key, _, _, _, _, _) => key.as_slice(), // Reaching here would require: // - the `execute_with_options` allowlist accepted a node // that doesn't carry a key (only `HashWithCount` / @@ -579,6 +611,7 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: vh, + reference_element_hash: None, }) } Node::KVCountSum(key, value, _, _) => { @@ -604,6 +637,7 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: vh, + reference_element_hash: None, }) } Node::KVValueHashFeatureType(key, value, vh, _) => { @@ -677,6 +711,113 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: *vh, + reference_element_hash: None, + }) + } + // ─── Resolved-reference returns ────────────────────────────── + // + // GroveDB's post-pass rewrote a reference row into one of these + // before encoding, so `value` is the RESOLVED target's bytes and + // the node's hash field is the REFERENCE element's own value + // hash. Phase 1 already bound both together: the tree-hash + // reconstruction for these variants computes + // `combine_hash(reference_element_hash, H(value))`, so a forged + // target value or a forged reference hash breaks the root. + // + // Note the KV→KVValueHash forgery guard that `KVValueHashFeatureType` + // needs does NOT apply here, and its absence is not a gap: that + // guard exists because a proof-carried `value_hash` is not checked + // against `H(value)`. For these variants the value hash IS + // recomputed from the value bytes as part of the combine, so a + // substituted `value` cannot survive. + Node::KVRefValueHashCount(key, value, reference_element_hash, _) => { + if !in_range { + return Err(Error::InvalidProofError( + "count-offset proof: KVRefValueHashCount at an out-of-range position" + .to_string(), + )); + } + if own_count != 1 { + return Err(Error::InvalidProofError(format!( + "count-offset proof: KVRefValueHashCount at own_count={} (expected 1)", + own_count + ))); + } + Ok(BoundaryKind::ValueReturned { + key: key.as_slice(), + value: value.as_slice(), + // The committed value hash for a combined reference is + // `combine_hash(reference_element_hash, H(target))` — + // recomputed here so the surfaced hash is the one the + // secondary root actually binds, not just half of it. + value_hash: crate::tree::combine_hash( + reference_element_hash, + &compute_value_hash(value.as_slice()).unwrap(), + ) + .unwrap(), + reference_element_hash: Some(*reference_element_hash), + }) + } + Node::KVRefValueHashCountSum(key, value, reference_element_hash, _, _) => { + if !in_range { + return Err(Error::InvalidProofError( + "count-offset proof: KVRefValueHashCountSum at an out-of-range position" + .to_string(), + )); + } + if own_count != 1 { + return Err(Error::InvalidProofError(format!( + "count-offset proof: KVRefValueHashCountSum at own_count={} (expected 1)", + own_count + ))); + } + Ok(BoundaryKind::ValueReturned { + key: key.as_slice(), + value: value.as_slice(), + value_hash: crate::tree::combine_hash( + reference_element_hash, + &compute_value_hash(value.as_slice()).unwrap(), + ) + .unwrap(), + reference_element_hash: Some(*reference_element_hash), + }) + } + Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + reference_element_hash, + _, + _, + target_child_hash, + ) => { + if !in_range { + return Err(Error::InvalidProofError( + "count-offset proof: KVRefValueHashCountSumWithTargetChildHash at an \ + out-of-range position" + .to_string(), + )); + } + if own_count != 1 { + return Err(Error::InvalidProofError(format!( + "count-offset proof: KVRefValueHashCountSumWithTargetChildHash at \ + own_count={} (expected 1)", + own_count + ))); + } + // Two combines, mirroring `Tree::hash` for this variant: the + // target is layered, so its committed hash folds its child + // commitment in before the reference folds over it. + let target_committed = crate::tree::combine_hash( + &compute_value_hash(value.as_slice()).unwrap(), + target_child_hash, + ) + .unwrap(); + Ok(BoundaryKind::ValueReturned { + key: key.as_slice(), + value: value.as_slice(), + value_hash: crate::tree::combine_hash(reference_element_hash, &target_committed) + .unwrap(), + reference_element_hash: Some(*reference_element_hash), }) } Node::KVValueHash(key, value, _) => { @@ -738,6 +879,10 @@ enum BoundaryKind<'a> { /// proof-carried value_hash (tree-flavored entries store /// `combine_hash(H(value), child_root)`). value_hash: CryptoHash, + /// For a resolved-reference return, the reference element's own + /// value hash; `None` for a directly-valued entry. See + /// [`CountOffsetReturnedItem::reference_element_hash`]. + reference_element_hash: Option, }, } @@ -779,6 +924,7 @@ fn apply_self_state(disposition: &BoundaryKind<'_>, state: &mut VerifyState) -> key, value, value_hash, + reference_element_hash, } => { if state.offset_remaining > 0 { return Err(Error::InvalidProofError( @@ -811,6 +957,7 @@ fn apply_self_state(disposition: &BoundaryKind<'_>, state: &mut VerifyState) -> // right behavior given that we don't carry a child // hash to validate. child_hash_verified: false, + reference_element_hash: *reference_element_hash, }); Ok(()) } diff --git a/merk/src/proofs/query/verify.rs b/merk/src/proofs/query/verify.rs index e5e4a92c9..6f26e2706 100644 --- a/merk/src/proofs/query/verify.rs +++ b/merk/src/proofs/query/verify.rs @@ -221,6 +221,8 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} + Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} + Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // cannot verify lower bound - we have an abridged // tree, so we cannot tell what the preceding key was @@ -263,6 +265,8 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} + Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} + Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // cannot verify upper bound - we have an abridged // tree so we cannot tell what the previous key was @@ -591,6 +595,26 @@ impl QueryProofVerify for Query { } execute_node(key, Some(value), *value_hash, false)?; } + Node::KVRefValueHashCountSumWithTargetChildHash( + key, + value, + value_hash, + _count, + _sum, + _target_child_hash, + ) => { + #[cfg(feature = "proof_debug")] + { + println!("Processing KVRefValueHashCountSumWithTargetChildHash node"); + } + // `child_hash_verified = true`: unlike the plain ref + // variant, the hash reconstruction for this node folds + // the target's child commitment in explicitly (see + // `Tree::hash`), so by the time the root matches, the + // target's layered binding has been independently + // checked — which is exactly what the flag means. + execute_node(key, Some(value), *value_hash, true)?; + } } last_push = Some(node.clone()); @@ -634,6 +658,7 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} + Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // proof contains abridged data so we cannot verify absence of // remaining query items diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index bf65dd62a..bedf0c3c4 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -364,6 +364,39 @@ impl Tree { ) }) } + Node::KVRefValueHashCountSumWithTargetChildHash( + key, + referenced_value, + node_value_hash, + count, + sum, + target_child_hash, + ) => { + // Two combines rather than one: the target is layered, so + // its own committed value hash is + // `combine_hash(H(target), target_child_hash)` — not + // `H(target)`. Reconstructing that first, then folding the + // reference over it, is what binds BOTH the target's bytes + // and its child commitment into the row's hash. Substituting + // either one breaks the root. + let mut cost = OperationCost::default(); + let referenced_value_hash = + value_hash(referenced_value.as_slice()).unwrap_add_cost(&mut cost); + let target_committed_hash = combine_hash(&referenced_value_hash, target_child_hash) + .unwrap_add_cost(&mut cost); + let combined_value_hash = combine_hash(node_value_hash, &target_committed_hash) + .unwrap_add_cost(&mut cost); + + kv_digest_to_kv_hash(key.as_slice(), &combined_value_hash).flat_map(|kv_hash| { + node_hash_with_count_and_sum( + &kv_hash, + &self.child_hash(true), + &self.child_hash(false), + *count, + *sum, + ) + }) + } } } @@ -541,7 +574,8 @@ impl Tree { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) => Some(key.as_slice()), + | Node::KVRefValueHashCountSum(key, ..) + | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.as_slice()), // These nodes don't have keys, only hashes Node::Hash(_) | Node::KVHash(_) From e5a48266460cafe83c35ba8f00f1bc925e54e000 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 03:09:48 +0700 Subject: [PATCH 02/18] feat: authenticate canonical axis rows and resolve them in proofs (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the proof and verification half of the reference-row change. **Axis proofs authenticate the canonical row, rather than assuming it.** A verifier never sees a row's reference bytes — only its committed reference hash — so previously it would have been trusting that a committed reference points at the key encoded in the row it sits in. It no longer has to: from the AUTHENTICATED primary value the verifier re-derives the (count, sum) the mirror would have seen, rebuilds both the secondary key and the canonical row those aggregates imply, and compares them against what the proof committed. One comparison covers the ordering prefix, the primary-key suffix, the reference path, the one-hop budget and the carried payload sum. A row filed under `…‖a` whose reference points at `b` cannot verify. Notably this needs no new wire field: everything it uses is already bound to the secondary root. Chain checks now run before row decoding, so a relabeled envelope is still reported as "not for the requested axis" rather than as the downstream row symptom. **Corruption coverage** for each way a row can be wrong, each with its own sentinel: legacy placeholder payload, plain Reference, non-sibling reference, wrong hop budget, wrong target, wrong payload sum, stale commitment. Plus the two write paths that must refresh a row without any aggregate moving (value-only update, deep mutation under a tree entry), a proof-level mis-targeted-row test, and a guard that ordinary user reference chains still resolve to their terminal. **Docs** updated to describe the reference-backed system: the book's secondary-row layout and proof walkthrough, the stale `resolve_values` design text (resolution is now normal read behaviour), the two count-offset comments that claimed a reference post-pass already existed, and the hop=1 well-formed-user contract, which now says explicitly that it governs ordinary references while indexed rows use the immediate-node rule through their own path. Deferred, deliberately: surfacing the resolved Element through the public read and result types (issue #814 Phase 3). The proof authenticates the value today and then drops it, because threading it out is ~500 call sites of pure API churn across 20 test files with no security content — better as its own reviewable change. The authentication itself was not deferrable and is here. Full workspace suite passes (43 targets); clippy warnings one below the pre-change baseline. --- docs/book/src/count-indexed-tree.md | 82 ++- grovedb/src/batch/mod.rs | 12 + .../indexed_axis/reference_resolution.rs | 4 +- .../operations/proof/indexed_axis/verify.rs | 320 +++++---- grovedb/src/operations/proof/verify.rs | 17 +- .../src/tests/indexed_reference_row_tests.rs | 610 ++++++++++++++++++ grovedb/src/tests/mod.rs | 1 + merk/src/proofs/query/count_offset/mod.rs | 15 +- merk/src/proofs/query/count_offset/verify.rs | 7 +- 9 files changed, 923 insertions(+), 145 deletions(-) create mode 100644 grovedb/src/tests/indexed_reference_row_tests.rs diff --git a/docs/book/src/count-indexed-tree.md b/docs/book/src/count-indexed-tree.md index 56564f548..e7ee5ff38 100644 --- a/docs/book/src/count-indexed-tree.md +++ b/docs/book/src/count-indexed-tree.md @@ -152,19 +152,64 @@ The secondary Merk holds one entry per element in the primary, keyed by: ```text secondary_key = count_be_bytes(8) ‖ original_key -secondary_val = () // empty; the original_key is encoded in the key +secondary_val = ReferenceWithSumItem( + SiblingReference(original_key), + max_reference_hop = Some(1), + sum = count_value, + ) ``` - **`count_be_bytes`** is the element's `count_value` encoded big-endian, 8 bytes. Big-endian gives natural numeric order under lexicographic comparison, so right-to-left iteration yields highest-count-first. - **`original_key`** is appended to break ties among elements with equal - counts and to make each secondary key unique and reversible. + counts and to make each secondary key unique and reversible. It stays + in the key even though the row also names it: the key is what orders + and de-duplicates, and it has to be decodable on its own. -The secondary Merk uses node feature type `ProvableCountedMerkNode(1)` — -every entry contributes a count of `1`, so the aggregated count at the -secondary's root equals the total number of indexed entries (which also -equals the number of entries in the primary). + + +### Secondary rows + +A row is a **canonical one-hop reference back to its primary entry**, +written as a *combined* reference so the row's committed value hash is + +```text +combine_hash(H(reference bytes), primary_node_committed_value_hash) +``` + +Three consequences worth stating plainly: + +- **Reads and proofs return the primary value.** The reference is + resolved and authenticated as part of the axis proof, so a top-k result + carries the values, not just pointers to them. +- **The binding is to the IMMEDIATE primary node**, not to a terminal + reached by following a chain. That keeps the invariant local: the only + thing that can staleness a row is a write to the primary entry itself, + which is exactly the event the mirror is driven by. This is dedicated + indexed-tree behaviour — ordinary GroveDB references keep their normal + terminal semantics, and an ordinary `max_hop = 1` reference pointing at + another reference remains ill-formed. +- **Value-only updates now write.** Because the row binds a commitment, + an update that changes a primary entry's bytes without moving its + `count_value` still rewrites the row on every configured axis. So does + a deep mutation that only moves a child subtree's root. This write + amplification is intentional and is charged in the cost estimates. + +`SiblingReference` rather than an absolute path keeps a row's size +independent of how deep the grove is. The reference is interpreted +against the row's **logical origin** — the indexed primary's path — not +against the derived storage prefix the secondary physically lives under. +That prefix is `blake3(primary_prefix ‖ axis_tag)` and is not a GroveDB +path at all, so resolution of a row's reference is purpose-built +machinery rather than the ordinary path-keyed reference following. + +The secondary Merk uses node feature type +`ProvableCountedAndProvableSummedMerkNode(1, count_value)` — every entry +contributes a count of `1`, so the aggregated count at the secondary's +root equals the total number of indexed entries (which also equals the +number of entries in the primary), while the sum half makes a band TOTAL +answerable as one committed scalar. The reason the secondary is a *provable* count tree (rather than the simpler `BasicMerkNode`) is that this lets the existing @@ -439,11 +484,7 @@ let result = GroveDb::verify_indexed_count_top_k(&proof_bytes, &path, k)?; // result.entries: Vec<(u64, Vec)>, result.root_hash: [u8; 32] ``` -The query returns `(count, key)` pairs. To resolve a primary value the -caller follows up with `db.get(path, key, ...)`; the dedicated proof -shape carries only the secondary range proof + a 32-byte attestation -of the primary's root hash. Workloads that don't need values -(leaderboards, ranking views) pay nothing for data they wouldn't read. +The query returns `(count, key)` pairs. Internally: @@ -453,14 +494,17 @@ Internally: 3. Run a **descending range query** with `limit = k` over the full secondary keyspace. This yields the k highest-count entries, with a standard Merk range proof. -4. *(only if `resolve_values: true`)* For each `(c_be ‖ k)` in the - result, open the **primary** Merk and query for `k`. Each resolution - is one extra Merk read with one extra Merk inclusion proof. - -The default keeps the proof minimal: secondary range proof + a 32-byte -attestation of the primary's root hash. Workloads that don't need the -values (leaderboards, ranking views, "top N usernames") pay nothing for -data they wouldn't read. +4. Resolve each returned row through its reference and emit a + reference-aware proof node carrying the primary value. This costs no + extra Merk inclusion proof: a row IS a reference to its primary + entry, and the resolved value is folded into the row's own committed + hash (see [Secondary rows](#secondary-rows)). + +The verifier authenticates the resolved value, and with it the row's +reference path, hop budget and carried sum, by rebuilding the canonical +row the primary value implies and comparing it against what the proof +committed. A row filed under one key whose reference points at another +cannot verify. ### Range by count diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7a8f608ba..08127fa13 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1749,6 +1749,18 @@ where // hash mismatch that `verify_grovedb` later reports. The // contract is the user's to uphold; we don't pay the price of // an extra dispatch on every well-formed hop=1 ref. + // + // That contract governs ORDINARY user references only. Indexed + // secondary rows are also one-hop, and for them binding the + // immediate target's merk-stored hash — whatever its shape — is + // the CANONICAL rule, not an ill-formed state: a row is meant to + // commit its primary entry's node, so a tree- or + // reference-shaped primary is expected, and + // `verify_indexed_axis_content` checks rows against that rule + // instead of the terminal-reference one. Indexed rows are + // written by the mirror through its own path, so they do not + // travel through here; the two rules stay separate, and neither + // is ever inferred from `max_reference_hop == 1` alone. if recursions_allowed == 1 { let merk = match self.merks.entry(reference_path.to_vec()) { HashMapEntry::Occupied(o) => o.into_mut(), diff --git a/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs b/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs index dc70b3553..c30f6c7b7 100644 --- a/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs +++ b/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs @@ -43,9 +43,7 @@ //! [`INDEXED_SECONDARY_MAX_HOP`]: //! crate::operations::indexed_tree::INDEXED_SECONDARY_MAX_HOP -use grovedb_costs::{ - cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, -}; +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; use grovedb_element::indexed::IndexAxis; use grovedb_merk::{ element::{costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions}, diff --git a/grovedb/src/operations/proof/indexed_axis/verify.rs b/grovedb/src/operations/proof/indexed_axis/verify.rs index 16f0d6328..c2f55befe 100644 --- a/grovedb/src/operations/proof/indexed_axis/verify.rs +++ b/grovedb/src/operations/proof/indexed_axis/verify.rs @@ -6,10 +6,7 @@ //! from a hash it has already verified, layer by layer, until the //! reconstructed GroveDB root hash is returned for the caller to compare. -use grovedb_element::indexed::{ - decode_count_sort_key, decode_sum_sort_key, encode_count_sort_key, encode_sum_sort_key, - IndexAxis, -}; +use grovedb_element::indexed::{encode_count_sort_key, encode_sum_sort_key, IndexAxis}; use grovedb_merk::{ proofs::{ query::{ @@ -24,7 +21,7 @@ use grovedb_merk::{ use grovedb_query::{AggregateFold, QueryItem as MerkQueryItem}; use grovedb_version::{check_grovedb_v0, version::GroveVersion}; -use crate::{Error, GroveDb}; +use crate::{Element, Error, GroveDb}; use super::{ aggregate_range_out_of_domain, AncestorAttestation, AxisEntries, IndexedAxisAggregateProof, @@ -342,7 +339,7 @@ impl GroveDb { let mut full_range = MerkQuery::new(); full_range.insert_all(); full_range.left_to_right = !envelope.descending; - verify_indexed_axis_range_inner(envelope, full_range, expected_axis, path) + verify_indexed_axis_range_inner(envelope, full_range, expected_axis, path, grove_version) } /// Verify an `IndexedAxisRangeProof`-shaped arbitrary-query proof. @@ -389,7 +386,13 @@ impl GroveDb { expected_descending, envelope.descending ))); } - verify_indexed_axis_range_inner(envelope, secondary_query, expected_axis, path) + verify_indexed_axis_range_inner( + envelope, + secondary_query, + expected_axis, + path, + grove_version, + ) } /// Verify an `IndexedAxisPaginatedProof`-shaped paginated proof. @@ -444,7 +447,7 @@ impl GroveDb { expected_offset, envelope.requested_offset ))); } - verify_indexed_axis_paginated_inner(envelope, expected_axis, path) + verify_indexed_axis_paginated_inner(envelope, expected_axis, path, grove_version) } /// Verify a rank-of-key proof produced by @@ -609,6 +612,7 @@ fn verify_indexed_axis_range_inner( secondary_query: MerkQuery, axis: IndexAxis, path: &[&[u8]], + grove_version: &GroveVersion, ) -> Result { if envelope.layer_proofs.len() != path.len() { return Err(Error::CorruptedData(format!( @@ -639,8 +643,10 @@ fn verify_indexed_axis_range_inner( )) })?; - let entries = decode_axis_entries_from_result_set(axis, &sec_result.result_set)?; - + // Chain checks BEFORE row decoding. Both reject a relabeled or + // rebound envelope, but the chain check names the actual defect + // ("this proof is not for the axis you asked about") where a row + // check would only report the downstream symptom. let initial_root = verify_deepest_layer( &envelope.layer_proofs, path, @@ -660,6 +666,8 @@ fn verify_indexed_axis_range_inner( "indexed-axis range proof", )?; + let entries = decode_axis_entries_from_result_set(axis, &sec_result.result_set, grove_version)?; + Ok(IndexedAxisQueryResult { root_hash, entries }) } @@ -667,6 +675,7 @@ fn verify_indexed_axis_paginated_inner( envelope: IndexedAxisPaginatedProof, axis: IndexAxis, path: &[&[u8]], + grove_version: &GroveVersion, ) -> Result { if envelope.layer_proofs.len() != path.len() { return Err(Error::CorruptedData(format!( @@ -701,8 +710,11 @@ fn verify_indexed_axis_paginated_inner( "indexed-axis paginated proof: secondary count-offset proof failed to verify: {e}" )) })?; - let entries = - decode_axis_entries_from_count_offset_items(axis, &count_offset_result.returned_items)?; + let entries = decode_axis_entries_from_count_offset_items( + axis, + &count_offset_result.returned_items, + grove_version, + )?; let (secondary_root_hash, skipped) = (count_offset_result.root_hash, count_offset_result.skipped); @@ -817,119 +829,203 @@ fn verify_indexed_axis_aggregate_inner( }) } -pub(crate) fn decode_axis_entries_from_result_set( +/// Authenticate ONE resolved secondary row and decode it. +/// +/// Everything needed is already in the proof, and all of it is bound to +/// the secondary root: the row's key, its committed reference hash, and +/// the resolved target bytes. From the AUTHENTICATED target element we +/// re-derive the `(count, sum)` the mirror would have seen, rebuild both +/// the secondary key and the canonical row those aggregates imply, and +/// check them against what the proof carried. +/// +/// That single comparison authenticates every part of the row at once: +/// +/// - the ordering prefix (it is part of the rebuilt secondary key), +/// - the primary key in the suffix (likewise), +/// - the reference path — that it is `SiblingReference(primary_key)` and +/// not a reference to some other entry, +/// - the one-hop budget, +/// - the carried payload sum. +/// +/// Without it a verifier would be trusting that a committed reference +/// points at the key encoded in the row it sits in. It does not have to: +/// a row filed under `…‖a` whose reference points at `b` produces a +/// different canonical-row hash, and is rejected here. +fn authenticate_axis_row( axis: IndexAxis, - result_set: &[grovedb_merk::proofs::query::ProvedKeyOptionalValue], + secondary_key: &[u8], + target_bytes: &[u8], + reference_element_hash: grovedb_merk::CryptoHash, + grove_version: &GroveVersion, +) -> Result<(Vec, Element), Error> { + use grovedb_merk::tree::value_hash; + + use crate::operations::indexed_tree::{ + axis_payload_sum, axis_row_reference, axis_sort_key_len, make_axis_secondary_key, + }; + + let sort_len = axis_sort_key_len(axis); + if secondary_key.len() < sort_len { + return Err(Error::CorruptedData(format!( + "indexed-axis ({axis:?}) secondary key shorter than {sort_len} bytes: {}", + hex::encode(secondary_key) + ))); + } + let primary_key = secondary_key[sort_len..].to_vec(); + + let element = Element::deserialize(target_bytes, grove_version).map_err(|e| { + Error::CorruptedData(format!( + "indexed-axis proof returned a non-Element primary value at key {}: {e}", + hex::encode(&primary_key) + )) + })?; + let (count, sum) = element.count_sum_value_or_default(); + + let expected_secondary_key = make_axis_secondary_key(axis, count, sum, &primary_key); + if expected_secondary_key != secondary_key { + return Err(Error::CorruptedData(format!( + "indexed-axis ({axis:?}) row is filed under {} but the primary value it resolves \ + to implies {} — the row's sort position does not match its own value", + hex::encode(secondary_key), + hex::encode(&expected_secondary_key) + ))); + } + + let expected_row = axis_row_reference(axis, &primary_key, count, sum)?; + let expected_row_bytes = expected_row.serialize(grove_version).map_err(|e| { + Error::CorruptedData(format!("serializing the expected canonical axis row: {e}")) + })?; + let expected_reference_hash = value_hash(&expected_row_bytes).unwrap(); + if expected_reference_hash != reference_element_hash { + return Err(Error::CorruptedData(format!( + "indexed-axis ({axis:?}) row at {} does not commit the canonical reference for its \ + primary key — expected a one-hop SiblingReference({}) carrying sum {}", + hex::encode(secondary_key), + hex::encode(&primary_key), + axis_payload_sum(axis, count, sum)? + ))); + } + + Ok((primary_key, element)) +} + +/// Build the per-axis entry list from authenticated `(secondary_key, +/// target_bytes, reference_element_hash)` triples. +/// +/// The resolved primary Element is authenticated here and then dropped: +/// the public result type still carries `(ordering_value, primary_key)`. +/// Surfacing the resolved value through the read and result APIs is a +/// separate, purely mechanical change (issue #814 Phase 3) and is +/// deliberately not bundled with the representation change — but the +/// authentication below is NOT deferrable, because without it a verifier +/// would be assuming that a committed reference points at the key encoded +/// in the row it sits in. +fn axis_entries_from_authenticated_rows( + axis: IndexAxis, + rows: impl IntoIterator, Vec, grovedb_merk::CryptoHash)>, + grove_version: &GroveVersion, ) -> Result { - match axis { - IndexAxis::Count => { - let mut entries: Vec<(u64, Vec)> = Vec::with_capacity(result_set.len()); - for proved in result_set { - if proved.key.len() < 8 { - return Err(Error::CorruptedData(format!( - "indexed-axis (count) secondary key shorter than 8 bytes: {:?}", - proved.key - ))); - } - let mut count_bytes = [0u8; 8]; - count_bytes.copy_from_slice(&proved.key[..8]); - entries.push(( - decode_count_sort_key(&count_bytes), - proved.key[8..].to_vec(), - )); + use grovedb_element::indexed::sort_keys::{ + decode_avg_sort_key, decode_count_sort_key, decode_sum_sort_key, + }; + + let mut count_entries: Vec<(u64, Vec)> = Vec::new(); + let mut sum_entries: Vec<(i64, Vec)> = Vec::new(); + let mut avg_entries: Vec<(i128, Vec)> = Vec::new(); + + for (secondary_key, target_bytes, reference_element_hash) in rows { + let (primary_key, value) = authenticate_axis_row( + axis, + &secondary_key, + &target_bytes, + reference_element_hash, + grove_version, + )?; + match axis { + IndexAxis::Count => { + let mut b = [0u8; 8]; + b.copy_from_slice(&secondary_key[..8]); + let _ = value; + count_entries.push((decode_count_sort_key(&b), primary_key)); } - Ok(AxisEntries::Count(entries)) - } - IndexAxis::Sum => { - let mut entries: Vec<(i64, Vec)> = Vec::with_capacity(result_set.len()); - for proved in result_set { - if proved.key.len() < 8 { - return Err(Error::CorruptedData(format!( - "indexed-axis (sum) secondary key shorter than 8 bytes: {:?}", - proved.key - ))); - } - let mut sum_bytes = [0u8; 8]; - sum_bytes.copy_from_slice(&proved.key[..8]); - entries.push((decode_sum_sort_key(&sum_bytes), proved.key[8..].to_vec())); + IndexAxis::Sum => { + let mut b = [0u8; 8]; + b.copy_from_slice(&secondary_key[..8]); + let _ = value; + sum_entries.push((decode_sum_sort_key(&b), primary_key)); } - Ok(AxisEntries::Sum(entries)) - } - IndexAxis::Avg => { - let mut entries: Vec<(i128, Vec)> = Vec::with_capacity(result_set.len()); - for proved in result_set { - if proved.key.len() < 16 { - return Err(Error::CorruptedData(format!( - "indexed-axis (avg) secondary key shorter than 16 bytes: {:?}", - proved.key - ))); - } - let mut avg_bytes = [0u8; 16]; - avg_bytes.copy_from_slice(&proved.key[..16]); - entries.push(( - grovedb_element::indexed::decode_avg_sort_key(&avg_bytes), - proved.key[16..].to_vec(), - )); + IndexAxis::Avg => { + let mut b = [0u8; 16]; + b.copy_from_slice(&secondary_key[..16]); + let _ = value; + avg_entries.push((decode_avg_sort_key(&b), primary_key)); } - Ok(AxisEntries::Avg(entries)) } } + + Ok(match axis { + IndexAxis::Count => AxisEntries::Count(count_entries), + IndexAxis::Sum => AxisEntries::Sum(sum_entries), + IndexAxis::Avg => AxisEntries::Avg(avg_entries), + }) } +/// Decode + authenticate the rows of an ordinary (non-count-offset) axis +/// range proof. +/// +/// `proved.proof` is the value hash the merk verifier surfaced for the +/// node. For the resolved-reference nodes an axis proof emits, that is +/// the REFERENCE element's own hash — which is exactly what +/// [`authenticate_axis_row`] needs. A row that was not a resolved +/// reference fails that check, which is the correct outcome: every +/// indexed secondary row must be one. +pub(crate) fn decode_axis_entries_from_result_set( + axis: IndexAxis, + result_set: &[grovedb_merk::proofs::query::ProvedKeyOptionalValue], + grove_version: &GroveVersion, +) -> Result { + let rows = result_set + .iter() + .map(|proved| { + let value = proved.value.clone().ok_or_else(|| { + Error::CorruptedData(format!( + "indexed-axis ({axis:?}) proof returned no value for secondary key {} — \ + every row resolves to its primary entry", + hex::encode(&proved.key) + )) + })?; + Ok((proved.key.clone(), value, proved.proof)) + }) + .collect::, Error>>()?; + axis_entries_from_authenticated_rows(axis, rows, grove_version) +} + +/// Decode + authenticate the rows of a count-offset paginated axis proof. +/// +/// `reference_element_hash` is `Some` exactly when the prover's reference +/// post-pass rewrote the row into a resolved-value node. `None` means the +/// row was surfaced without being resolved, which for an indexed +/// secondary is malformed. pub(crate) fn decode_axis_entries_from_count_offset_items( axis: IndexAxis, items: &[grovedb_merk::proofs::query::CountOffsetReturnedItem], + grove_version: &GroveVersion, ) -> Result { - match axis { - IndexAxis::Count => { - let mut entries: Vec<(u64, Vec)> = Vec::with_capacity(items.len()); - for it in items { - if it.key.len() < 8 { - return Err(Error::CorruptedData(format!( - "indexed-axis (count) paginated secondary key shorter than 8 bytes: {:?}", - it.key - ))); - } - let mut count_bytes = [0u8; 8]; - count_bytes.copy_from_slice(&it.key[..8]); - entries.push((decode_count_sort_key(&count_bytes), it.key[8..].to_vec())); - } - Ok(AxisEntries::Count(entries)) - } - IndexAxis::Avg => { - let mut entries: Vec<(i128, Vec)> = Vec::with_capacity(items.len()); - for it in items { - if it.key.len() < 16 { - return Err(Error::CorruptedData(format!( - "indexed-axis (avg) paginated secondary key shorter than 16 bytes: {:?}", - it.key - ))); - } - let mut avg_bytes = [0u8; 16]; - avg_bytes.copy_from_slice(&it.key[..16]); - entries.push(( - grovedb_element::indexed::decode_avg_sort_key(&avg_bytes), - it.key[16..].to_vec(), - )); - } - Ok(AxisEntries::Avg(entries)) - } - IndexAxis::Sum => { - let mut entries: Vec<(i64, Vec)> = Vec::with_capacity(items.len()); - for it in items { - if it.key.len() < 8 { - return Err(Error::CorruptedData(format!( - "indexed-axis (sum) paginated secondary key shorter than 8 bytes: {:?}", - it.key - ))); - } - let mut sum_bytes = [0u8; 8]; - sum_bytes.copy_from_slice(&it.key[..8]); - entries.push((decode_sum_sort_key(&sum_bytes), it.key[8..].to_vec())); - } - Ok(AxisEntries::Sum(entries)) - } - } + let rows = items + .iter() + .map(|it| { + let reference_element_hash = it.reference_element_hash.ok_or_else(|| { + Error::CorruptedData(format!( + "indexed-axis ({axis:?}) paginated proof surfaced an unresolved row at \ + secondary key {} — every row is a canonical reference and must arrive \ + resolved", + hex::encode(&it.key) + )) + })?; + Ok((it.key.clone(), it.value.clone(), reference_element_hash)) + }) + .collect::, Error>>()?; + axis_entries_from_authenticated_rows(axis, rows, grove_version) } pub(crate) fn count_aggregate_inner_range(lo: i128, hi: i128) -> MerkQueryItemForRange { diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 816abf7d1..eb8932db3 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -911,8 +911,11 @@ impl GroveDb { format!("axis descent: secondary count-offset proof failed: {e}"), ) })?; - let entries = - decode_axis_entries_from_count_offset_items(axis, &res.returned_items)?; + let entries = decode_axis_entries_from_count_offset_items( + axis, + &res.returned_items, + grove_version, + )?; ( res.root_hash, AxisWalkResult::Entries { @@ -957,8 +960,11 @@ impl GroveDb { ), )); } - let entries = - decode_axis_entries_from_count_offset_items(axis, &res.returned_items)?; + let entries = decode_axis_entries_from_count_offset_items( + axis, + &res.returned_items, + grove_version, + )?; let yielded_key = entries.first_original_key().map(|k| k.to_vec()); if yielded_key.as_deref() != Some(key.as_slice()) { return Err(Error::InvalidProof( @@ -1010,7 +1016,8 @@ impl GroveDb { format!("axis descent: secondary range proof failed: {e}"), ) })?; - let entries = decode_axis_entries_from_result_set(axis, &res.result_set)?; + let entries = + decode_axis_entries_from_result_set(axis, &res.result_set, grove_version)?; ( root, AxisWalkResult::Entries { diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs new file mode 100644 index 000000000..8e2cbe347 --- /dev/null +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -0,0 +1,610 @@ +//! Corruption and tamper coverage for canonical indexed-secondary rows. +//! +//! Every indexed secondary row is a canonical one-hop +//! `ReferenceWithSumItem(SiblingReference(primary_key), Some(1), +//! axis_payload_sum)` written as a COMBINED reference, so its committed +//! value hash is +//! `combine_hash(H(reference bytes), primary_node_value_hash)`. +//! +//! Each test here breaks exactly ONE part of that and asserts +//! `verify_grovedb` names it. The point of separate sentinels is that a +//! corruption report tells an operator which half is wrong — a stale +//! commitment (the primary moved without the mirror running) needs a +//! different response from a malformed reference (the row was written by +//! something that isn't the mirror). + +#[cfg(test)] +mod tests { + use grovedb_element::{indexed::IndexAxis, reference_path::ReferencePathType}; + use grovedb_merk::element::{ + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + }; + use grovedb_path::SubtreePath; + use grovedb_storage::{Storage, StorageBatch}; + use grovedb_version::version::GroveVersion; + + use crate::{ + operations::indexed_tree::make_axis_secondary_key, + tests::{make_test_grovedb, TEST_LEAF}, + Element, GroveDb, + }; + + /// A PCIT at `[TEST_LEAF, "cidx"]` holding one item-shaped entry + /// `"a"`, which is the shape whose row we then damage. + fn pcit_with_one_entry(grove_version: &GroveVersion) -> crate::tests::TempGroveDb { + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("create PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"a", + Element::new_item(b"v".to_vec()), + None, + grove_version, + ) + .unwrap() + .expect("insert entry"); + db + } + + /// Overwrite the row at the canonical secondary key with `row`, + /// bound to `target_hash`. Passing the honest target hash isolates a + /// row-CONTENT change; passing a different one isolates a + /// COMMITMENT change. + fn overwrite_row( + db: &GroveDb, + secondary_key: &[u8], + row: Element, + target_hash: Option<[u8; 32]>, + grove_version: &GroveVersion, + ) { + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let path_segments: [&[u8]; 2] = [TEST_LEAF, b"cidx".as_ref()]; + let path: SubtreePath<&[u8]> = (&path_segments).into(); + + let secondary_root_key = { + let parent_merk = db + .open_transactional_merk_at_path( + [TEST_LEAF].as_ref().into(), + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open parent"); + let cidx = Element::get(&parent_merk, b"cidx", true, grove_version) + .unwrap() + .expect("cidx element"); + match cidx.underlying() { + Element::ProvableCountIndexedTree(_, s, ..) => s.clone(), + other => panic!("not a PCIT element: {other:?}"), + } + }; + + { + let mut secondary = db + .open_indexed_secondary_at_path( + path, + IndexAxis::Count, + secondary_root_key, + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open secondary"); + // Bind to the honest primary node hash unless the caller is + // deliberately moving the commitment. + let bind_to = target_hash.unwrap_or_else(|| { + let primary = db + .open_transactional_merk_at_path( + [TEST_LEAF, b"cidx".as_ref()].as_ref().into(), + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open primary"); + primary + .get_value_hash( + b"a", + true, + None::<&fn(&[u8], &GroveVersion) -> _>, + grove_version, + ) + .unwrap() + .expect("value hash read") + .expect("entry present") + }); + row.insert_reference(&mut secondary, secondary_key, bind_to, None, grove_version) + .unwrap() + .expect("write row"); + } + + db.db + .commit_multi_context_batch(batch, Some(&tx)) + .unwrap() + .expect("commit"); + tx.commit().expect("tx commit"); + } + + /// The sentinel path `verify_grovedb` files an issue under for the + /// count axis of the PCIT built above. + fn sentinel_path(kind: &str) -> Vec> { + vec![ + TEST_LEAF.to_vec(), + b"cidx".to_vec(), + format!("__cidx_{kind}__").into_bytes(), + b"a".to_vec(), + ] + } + + fn assert_only_issue(db: &GroveDb, kind: &str, grove_version: &GroveVersion) { + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + let want = sentinel_path(kind); + assert!( + issues.contains_key(&want), + "expected a `__cidx_{kind}__` issue, got {:?}", + issues + .keys() + .map(|k| k + .iter() + .map(|s| String::from_utf8_lossy(s).to_string()) + .collect::>()) + .collect::>() + ); + } + + /// Baseline: an untouched tree verifies clean, and its row really is + /// the canonical shape. Without this the corruption tests below could + /// pass for the wrong reason. + #[test] + fn a_healthy_indexed_tree_stores_canonical_reference_rows() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!(issues.is_empty(), "healthy tree reported: {issues:?}"); + + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let path_segments: [&[u8]; 2] = [TEST_LEAF, b"cidx".as_ref()]; + let secondary_root_key = { + let parent_merk = db + .open_transactional_merk_at_path( + [TEST_LEAF].as_ref().into(), + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .unwrap(); + match Element::get(&parent_merk, b"cidx", true, grove_version) + .unwrap() + .unwrap() + .underlying() + { + Element::ProvableCountIndexedTree(_, s, ..) => s.clone(), + other => panic!("not a PCIT: {other:?}"), + } + }; + let secondary = db + .open_indexed_secondary_at_path( + (&path_segments).into(), + IndexAxis::Count, + secondary_root_key, + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .unwrap(); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + let row = Element::get(&secondary, key.as_slice(), true, grove_version) + .unwrap() + .expect("row present"); + assert_eq!( + row, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"a".to_vec()), + Some(1), + // The count axis carries the COUNT as its payload sum, so + // a band Total stays one committed scalar (#806). + 1, + ), + "a healthy count-axis row must be the canonical one-hop reference" + ); + } + + /// A legacy placeholder row — the representation this change + /// replaced — must be rejected, not silently accepted. + #[test] + fn a_legacy_placeholder_row_is_reported_as_non_canonical() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row(&db, &key, Element::new_sum_item(1), None, grove_version); + assert_only_issue(&db, "secondary_non_canonical_row", grove_version); + } + + /// A plain `Reference` folds to `(1, 0)` in a PCPS secondary, which + /// would silently zero the band Total. It is not canonical. + #[test] + fn a_plain_reference_row_is_reported_as_non_canonical() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference(ReferencePathType::SiblingReference(b"a".to_vec())), + None, + grove_version, + ); + assert_only_issue(&db, "secondary_non_canonical_row", grove_version); + } + + /// A non-sibling reference type would make row size grow with grove + /// depth and breaks the logical-origin rule. + #[test] + fn a_non_sibling_reference_row_is_reported_as_non_canonical() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"cidx".to_vec(), + b"a".to_vec(), + ]), + Some(1), + 1, + ), + None, + grove_version, + ); + assert_only_issue(&db, "secondary_non_canonical_row", grove_version); + } + + /// The hop budget is part of the binding rule: one hop means the row + /// binds the IMMEDIATE primary node. A different budget is a + /// different binding and must not be accepted as canonical. + #[test] + fn a_multi_hop_row_is_reported_as_non_canonical() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"a".to_vec()), + Some(2), + 1, + ), + None, + grove_version, + ); + assert_only_issue(&db, "secondary_non_canonical_row", grove_version); + } + + /// Canonical shape, but pointing at a different primary key than the + /// secondary-key suffix encodes. The suffix and the reference are two + /// independent encodings of the same fact and must agree. + #[test] + fn a_row_referencing_the_wrong_primary_key_is_reported() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"somewhere-else".to_vec()), + Some(1), + 1, + ), + None, + grove_version, + ); + assert_only_issue(&db, "secondary_wrong_reference_target", grove_version); + } + + /// Canonical shape and target, wrong carried sum. On the count axis + /// the payload sum is the COUNT, so a wrong one corrupts band totals + /// while leaving the sort position — and therefore every key-only + /// check — looking correct. + #[test] + fn a_row_with_the_wrong_payload_sum_is_reported() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"a".to_vec()), + Some(1), + 99, + ), + None, + grove_version, + ); + assert_only_issue(&db, "secondary_wrong_payload_sum", grove_version); + } + + /// The headline case for the reference representation: every BYTE of + /// the row is canonical and correct, and only the commitment is + /// stale. This is what a value-only primary update produces if the + /// mirror fails to refresh, and it is invisible to any check that + /// compares serialized rows alone. + #[test] + fn a_row_bound_to_a_stale_target_hash_is_reported() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + let canonical = Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"a".to_vec()), + Some(1), + 1, + ); + overwrite_row(&db, &key, canonical, Some([0xEE; 32]), grove_version); + assert_only_issue(&db, "secondary_stale_target_hash", grove_version); + } + + /// A value-only update must refresh the row's commitment through the + /// real write path — the case the pre-reference mirror was free to + /// skip because `(count, sum)` did not move. + /// + /// Driven through the public API on BOTH entry points, because the + /// direct and batch mirrors are separate implementations and a fix to + /// one does not imply the other. + #[test] + fn a_value_only_update_refreshes_the_row_on_both_write_paths() { + let grove_version = GroveVersion::latest(); + + // Direct path. + let db = pcit_with_one_entry(grove_version); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"a", + Element::new_item(b"a-completely-different-value".to_vec()), + None, + grove_version, + ) + .unwrap() + .expect("value-only update"); + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "direct value-only update left the row stale: {issues:?}" + ); + + // Batch path. + let db = pcit_with_one_entry(grove_version); + db.apply_batch( + vec![crate::batch::QualifiedGroveDbOp::replace_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec()], + b"a".to_vec(), + Element::new_item(b"another-completely-different-value".to_vec()), + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch value-only update"); + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "batch value-only update left the row stale: {issues:?}" + ); + } + + /// A deep mutation under a TREE-shaped entry moves that entry's + /// committed value hash (its child root changed) while leaving its + /// count alone. The row must be refreshed even though nothing about + /// its sort position moved. + /// + /// This reaches the mirror only via the synthesized propagation op on + /// the primary entry, which the aggregate-only mirror skipped as + /// unchanged — so it is a distinct path from the value-only case + /// above, not a restatement of it. + #[test] + fn a_deep_mutation_under_a_tree_entry_refreshes_the_row() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("create PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"sub", + Element::empty_tree(), + None, + grove_version, + ) + .unwrap() + .expect("tree child"); + // Populate it once so the entry has a non-null child root, then + // change that root WITHOUT changing the child's count. + db.insert( + [TEST_LEAF, b"cidx", b"sub"].as_ref(), + b"k", + Element::new_item(b"first".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("populate"); + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!(issues.is_empty(), "after populate: {issues:?}"); + + db.insert( + [TEST_LEAF, b"cidx", b"sub"].as_ref(), + b"k", + Element::new_item(b"second".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("deep value-only update"); + + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "a deep mutation that moved the child root left the row stale: {issues:?}" + ); + } + + /// An axis PROOF must not merely assume that a committed reference + /// points at the key encoded in the row it sits in. + /// + /// This is the row-level analogue of the `verify_grovedb` target + /// check above, and it needs its own coverage: `verify_grovedb` reads + /// the reference path directly out of storage, while a verifier never + /// sees those bytes — it only sees the row's committed reference + /// hash. The check works by rebuilding the canonical row from the + /// AUTHENTICATED primary value and comparing hashes, so a row whose + /// reference points elsewhere cannot survive it. + /// + /// Damaging the row makes the secondary root move, so the tampered + /// state is rejected either as a broken chain or as a non-canonical + /// row — both are correct refusals, and asserting on "rejected" + /// rather than on one message keeps the test from pinning which guard + /// happens to fire first. + #[test] + fn an_axis_proof_over_a_mis_targeted_row_is_rejected() { + let grove_version = GroveVersion::latest(); + let db = pcit_with_one_entry(grove_version); + // A second entry, so the mis-targeted reference points at a key + // that genuinely exists — the interesting case, since a dangling + // reference would fail earlier and for a duller reason. + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"b", + Element::new_item(b"other".to_vec()), + None, + grove_version, + ) + .unwrap() + .expect("second entry"); + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let honest = db + .prove_indexed_count_top_k(path, 4, true, None, grove_version) + .unwrap() + .expect("honest proof"); + GroveDb::verify_indexed_count_top_k(&honest, path, 4, true, grove_version) + .expect("the honest proof must verify"); + + // Point "a"'s row at "b" while leaving its sort position alone. + let key = make_axis_secondary_key(IndexAxis::Count, 1, 0, b"a"); + overwrite_row( + &db, + &key, + Element::new_reference_with_sum_item_with_hops( + ReferencePathType::SiblingReference(b"b".to_vec()), + Some(1), + 1, + ), + None, + grove_version, + ); + + let tampered = db + .prove_indexed_count_top_k(path, 4, true, None, grove_version) + .unwrap(); + let rejected = match tampered { + // The prover itself refuses to build a proof over a row it + // cannot square with its own key suffix. + Err(_) => true, + Ok(bytes) => { + GroveDb::verify_indexed_count_top_k(&bytes, path, 4, true, grove_version).is_err() + } + }; + assert!( + rejected, + "a row whose reference points at a different primary key than its own \ + secondary-key suffix must not produce a verifiable proof" + ); + } + + /// Ordinary user references keep their own semantics. The one-hop + /// immediate-node rule is dedicated indexed-tree behaviour, so it must + /// not leak into how a normal reference elsewhere in the grove is + /// treated: an ordinary reference still resolves to its TERMINAL. + #[test] + fn ordinary_user_references_are_unaffected() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_item(b"terminal-value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("target"); + db.insert( + [TEST_LEAF].as_ref(), + b"hop", + Element::new_reference(ReferencePathType::SiblingReference(b"target".to_vec())), + None, + None, + grove_version, + ) + .unwrap() + .expect("first hop"); + // A two-hop chain: an ordinary reference follows through to the + // terminal, which is exactly the semantics an indexed row does + // NOT use. + db.insert( + [TEST_LEAF].as_ref(), + b"entry", + Element::new_reference(ReferencePathType::SiblingReference(b"hop".to_vec())), + None, + None, + grove_version, + ) + .unwrap() + .expect("second hop"); + + let got = db + .get([TEST_LEAF].as_ref(), b"entry", None, grove_version) + .unwrap() + .expect("resolve"); + assert_eq!( + got, + Element::new_item(b"terminal-value".to_vec()), + "an ordinary reference chain must still resolve to its terminal" + ); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "ordinary references reported: {issues:?}" + ); + } +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index bafe8cfa7..830cdb6bf 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -60,6 +60,7 @@ mod indexed_axis_nested_and_bounds_tests; mod indexed_axis_offset_proof_tests; mod indexed_axis_paginated_cost_tests; mod indexed_axis_proof_tests; +mod indexed_reference_row_tests; mod indexed_tree_secondary_drift_tests; mod indexed_tree_security_regression_tests; mod is_empty_tree_tests; diff --git a/merk/src/proofs/query/count_offset/mod.rs b/merk/src/proofs/query/count_offset/mod.rs index e3e154132..007f2db1c 100644 --- a/merk/src/proofs/query/count_offset/mod.rs +++ b/merk/src/proofs/query/count_offset/mod.rs @@ -18,10 +18,17 @@ //! subtrees collapse to a single `HashWithCount` op (same as //! AggregateCountOnRange) so the offset region pays O(log n) proof size //! per skipped subtree rather than O(skipped). Returned items inside -//! the limit window emit as normal count-bearing value nodes (the same -//! `KVCount` / `KVRefValueHashCount` / etc. used by regular count-tree -//! proofs), so the result shape is byte-identical to what a regular -//! merk verifier would produce for the same range without offset. +//! the limit window emit as normal count-bearing value nodes (`KVCount` +//! / `KVCountSum` for directly-valued entries, `KVValueHashFeatureType` +//! for tree- and reference-shaped ones), so the result shape is +//! byte-identical to what a regular merk verifier would produce for the +//! same range without offset. +//! +//! Reference rows arrive here as `KVValueHashFeatureType` carrying the +//! REFERENCE's bytes. The layer above rewrites those into the +//! `KVRefValueHash{Count,CountSum}` family carrying the resolved target +//! — the same post-pass the regular count-tree flow runs — and this +//! verifier accepts and authenticates the rewritten variants. //! //! ## Why `ProvableCountSumTree` only commits the count (not the sum) //! diff --git a/merk/src/proofs/query/count_offset/verify.rs b/merk/src/proofs/query/count_offset/verify.rs index 727fa1587..902adc63e 100644 --- a/merk/src/proofs/query/count_offset/verify.rs +++ b/merk/src/proofs/query/count_offset/verify.rs @@ -75,10 +75,13 @@ pub struct CountOffsetReturnedItem { /// The element's serialized value bytes, as emitted by the prover. /// /// For a `KVRefValueHash*` node these are the RESOLVED target's - /// bytes, not the reference's — GroveDB's reference post-pass runs + /// bytes, not the reference's: the caller's reference post-pass runs /// before the proof is encoded, so by verify time the dereferencing /// has already happened and `reference_element_hash` is what records - /// that this row was a reference. + /// that this row was a reference. (This is a change from the earlier + /// contract, which said dereferencing happened at the GroveDB layer + /// after verification — it never did, and reference rows were + /// rejected outright instead.) pub value: Vec, /// The value-hash the proof's merk node committed for this entry. /// For `KVCount` nodes this is `H(value)` (the Item-flavored value From 5db11b6b3909eee95f666699a006333acf6e2416 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 03:17:06 +0700 Subject: [PATCH 03/18] fix: refresh canonical rows on every direct non-Merk append (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MMR fix generalised: all four direct non-Merk append APIs (MMR, commitment, bulk-append, dense) share a shape where the updated element is written straight into the primary Merk and propagation starts only afterwards, so the propagation walk — which mirrors entries it discovers as it climbs — never sees the entry that actually moved. That was genuinely a no-op under aggregate-only rows: a non-Merk child contributes a constant count of 1, so an append moved nothing the row was keyed or valued on. Under canonical rows it is not, because the append rewrites the entry's non-Merk root and therefore its commitment. Bulk-append and dense were each leaving one stale row behind. Extracts the per-entry mirror into `capture_indexed_entry_state` + `mirror_indexed_entry_and_seed` rather than repeating it four times, and covers all four APIs in one test — each as its own case, since each has its own copy of the write-then-propagate sequence. --- grovedb/src/lib.rs | 129 ++++++++++++++++++ grovedb/src/operations/bulk_append_tree.rs | 29 +++- grovedb/src/operations/commitment_tree.rs | 29 +++- grovedb/src/operations/dense_tree.rs | 29 +++- grovedb/src/operations/mmr_tree.rs | 115 ++++------------ .../src/tests/indexed_reference_row_tests.rs | 98 +++++++++++++ 6 files changed, 334 insertions(+), 95 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 8d0942145..77d478496 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -867,6 +867,135 @@ impl GroveDb { Ok(staged).wrap_with_cost(cost) } + /// Snapshot an entry's indexed state before a non-Merk append + /// rewrites it, returning `None` when the parent is not an indexed + /// primary (in which case there is nothing to mirror). + /// + /// Pairs with [`Self::mirror_indexed_entry_and_seed`]; see its docs + /// for why the direct non-Merk append APIs need this at all. + pub(crate) fn capture_indexed_entry_state<'db>( + primary_merk: &Merk>, + key: &[u8], + element: &Element, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + if !primary_merk.tree_type.is_indexed_primary() { + return Ok(None).wrap_with_cost(cost); + } + let value_hash = cost_return_on_error!( + &mut cost, + primary_merk + .get_value_hash( + key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let (count, sum) = element.count_sum_value_or_default(); + Ok(value_hash.map( + |value_hash| crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, + }, + )) + .wrap_with_cost(cost) + } + + /// Mirror one rewritten entry of an indexed primary into its axis + /// secondaries and produce the seeds propagation needs. + /// + /// The direct non-Merk append APIs (MMR, commitment, bulk-append, + /// dense) write the updated element straight into the primary Merk + /// and only then start propagating, so the propagation walk — which + /// mirrors entries it discovers as it climbs — never sees the entry + /// that actually moved. That was harmless while rows were + /// aggregate-only (a non-Merk child's count is a constant `1`, so its + /// row never moved) and is not harmless now: a canonical row binds the + /// primary node's commitment, and an append rewrites exactly that. + /// + /// Returns `(initial_deferred_secondary, initial_deferred_axes)`. The + /// two are not interchangeable — propagation rebuilds a single-axis + /// element from the former and a PCPSIT from the latter, and seeding + /// the wrong one leaves the other set for a later iteration that has + /// no indexed element to apply it to. + #[allow(clippy::type_complexity)] + pub(crate) fn mirror_indexed_entry_and_seed<'b, 'db, B: AsRef<[u8]>>( + &self, + primary_path: SubtreePath<'b, B>, + primary_merk: &Merk>, + key: &[u8], + old_state: Option, + transaction: &Transaction, + batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult< + ( + Option<(Hash, Option>)>, + Option>)>>, + ), + Error, + > { + let mut cost = OperationCost::default(); + if !primary_merk.tree_type.is_indexed_primary() { + return Ok((None, None)).wrap_with_cost(cost); + } + + let (grandparent_path, indexed_key) = cost_return_on_error_no_add!( + cost, + primary_path + .derive_parent() + .ok_or(Error::CorruptedCodeExecution( + "an indexed primary requires a grandparent holding its indexed element", + )) + ); + let indexed_element = { + let grandparent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + grandparent_path, + transaction, + Some(batch), + grove_version, + ) + ); + cost_return_on_error!( + &mut cost, + Element::get(&grandparent_merk, indexed_key, true, grove_version) + .map_err(Error::MerkError) + ) + }; + let is_multi_axis = matches!( + indexed_element.underlying(), + Element::ProvableCountProvableSumIndexedTree(..) + ); + let staged = cost_return_on_error!( + &mut cost, + self.mirror_indexed_primary_entry( + primary_path, + primary_merk, + &indexed_element, + key, + old_state, + transaction, + batch, + grove_version, + ) + ); + let seeds = if is_multi_axis { + (None, Some(staged)) + } else { + ( + staged.first().map(|(_, hash, key)| (*hash, key.clone())), + None, + ) + }; + Ok(seeds).wrap_with_cost(cost) + } + pub(crate) fn propagate_changes_with_transaction_with_initial_deferred<'b, B: AsRef<[u8]>>( &self, mut merk_cache: HashMap, Merk>, diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 50d13cccb..25fd76588 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -123,6 +123,14 @@ impl GroveDb { ) ); + // A canonical indexed secondary row binds this entry's committed + // value hash, and an append moves it while leaving `(count, sum)` + // alone. Snapshot before the rewrite; mirror after. + let old_indexed_state = cost_return_on_error!( + &mut cost, + GroveDb::capture_indexed_entry_state(&parent_merk, key, &element, grove_version) + ); + let updated_element = Element::new_bulk_append_tree(new_total_count, chunk_power, existing_flags); @@ -138,14 +146,33 @@ impl GroveDb { ); // 5. Propagate changes + // Mirror the moved entry into the primary's axis secondaries, + // then propagate seeded with the mirror's per-axis roots so the + // indexed element one level up is rebuilt over the NEW secondary + // state rather than re-read from stale root keys. + let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( + &mut cost, + self.mirror_indexed_entry_and_seed( + path.clone(), + &parent_merk, + key, + old_indexed_state, + tx.as_ref(), + &batch, + grove_version, + ) + ); + let mut merk_cache = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction( + self.propagate_changes_with_transaction_with_initial_deferred_axes( merk_cache, path, + initial_deferred_secondary, + initial_deferred_axes, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index b55a2f4e9..2ddee1c08 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -208,6 +208,14 @@ impl GroveDb { ) ); + // A canonical indexed secondary row binds this entry's committed + // value hash, and an append moves it while leaving `(count, sum)` + // alone. Snapshot before the rewrite; mirror after. + let old_indexed_state = cost_return_on_error!( + &mut cost, + GroveDb::capture_indexed_entry_state(&parent_merk, key, &element, grove_version) + ); + let updated_element = Element::new_commitment_tree(new_total_count, chunk_power, existing_flags); @@ -223,14 +231,33 @@ impl GroveDb { ); // 6. Propagate changes from parent upward + // Mirror the moved entry into the primary's axis secondaries, + // then propagate seeded with the mirror's per-axis roots so the + // indexed element one level up is rebuilt over the NEW secondary + // state rather than re-read from stale root keys. + let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( + &mut cost, + self.mirror_indexed_entry_and_seed( + path.clone(), + &parent_merk, + key, + old_indexed_state, + tx.as_ref(), + &batch, + grove_version, + ) + ); + let mut merk_cache = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction( + self.propagate_changes_with_transaction_with_initial_deferred_axes( merk_cache, path, + initial_deferred_secondary, + initial_deferred_axes, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/dense_tree.rs b/grovedb/src/operations/dense_tree.rs index d8116edd2..84477ce11 100644 --- a/grovedb/src/operations/dense_tree.rs +++ b/grovedb/src/operations/dense_tree.rs @@ -121,6 +121,14 @@ impl GroveDb { ) ); + // A canonical indexed secondary row binds this entry's committed + // value hash, and an append moves it while leaving `(count, sum)` + // alone. Snapshot before the rewrite; mirror after. + let old_indexed_state = cost_return_on_error!( + &mut cost, + GroveDb::capture_indexed_entry_state(&parent_merk, key, &element, grove_version) + ); + let updated_element = Element::new_dense_tree(new_count, height, existing_flags); cost_return_on_error_into!( @@ -135,15 +143,34 @@ impl GroveDb { ); // 5. Propagate changes + // Mirror the moved entry into the primary's axis secondaries, + // then propagate seeded with the mirror's per-axis roots so the + // indexed element one level up is rebuilt over the NEW secondary + // state rather than re-read from stale root keys. + let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( + &mut cost, + self.mirror_indexed_entry_and_seed( + path.clone(), + &parent_merk, + key, + old_indexed_state, + tx.as_ref(), + &batch, + grove_version, + ) + ); + let mut merk_cache: HashMap, Merk> = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction( + self.propagate_changes_with_transaction_with_initial_deferred_axes( merk_cache, path, + initial_deferred_secondary, + initial_deferred_axes, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 1cfcf81ed..e725fe554 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -146,37 +146,13 @@ impl GroveDb { let updated_element = Element::new_mmr_tree(new_mmr_size, existing_flags); - // If the parent is an indexed primary, snapshot this entry's state - // BEFORE the rewrite. An append leaves `(count, sum)` alone but - // moves the entry's committed value hash, which is exactly what a - // canonical secondary row binds — so without this the row is left - // pointing at a commitment that no longer exists. The propagation - // walk cannot cover it: the mutation lands here, at the START path, - // rather than at a level the walk climbs through. - let parent_is_indexed_primary = parent_merk.tree_type.is_indexed_primary(); - let old_indexed_state = if parent_is_indexed_primary { - let old_value_hash = cost_return_on_error!( - &mut cost, - parent_merk - .get_value_hash( - key, - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(Error::MerkError) - ); - let (count, sum) = element.count_sum_value_or_default(); - old_value_hash.map( - |value_hash| crate::operations::indexed_tree::IndexedEntryState { - count, - sum, - value_hash, - }, - ) - } else { - None - }; + // A canonical indexed secondary row binds this entry's committed + // value hash, and an append moves it while leaving `(count, sum)` + // alone. Snapshot before the rewrite; mirror after. + let old_indexed_state = cost_return_on_error!( + &mut cost, + GroveDb::capture_indexed_entry_state(&parent_merk, key, &element, grove_version) + ); // MMR root hash flows as the Merk child hash cost_return_on_error!( @@ -186,67 +162,22 @@ impl GroveDb { .map_err(|e| e.into()) ); - // 5. Mirror the moved entry into the primary's axis secondaries (if - // the parent is an indexed primary), then propagate upward, - // seeding the mirror's per-axis roots so the indexed element one - // level up is rebuilt over the NEW secondary state rather than - // re-read from stale root keys. - let initial_deferred = if parent_is_indexed_primary { - let (grandparent_path, indexed_key) = cost_return_on_error_no_add!( - cost, - path.derive_parent().ok_or(Error::CorruptedCodeExecution( - "an indexed primary requires a grandparent holding its indexed element", - )) - ); - let grandparent_merk = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - grandparent_path, - tx.as_ref(), - Some(&batch), - grove_version, - ) - ); - let indexed_element = cost_return_on_error!( - &mut cost, - Element::get(&grandparent_merk, indexed_key, true, grove_version) - .map_err(Error::MerkError) - ); - drop(grandparent_merk); - let is_multi_axis = matches!( - indexed_element.underlying(), - Element::ProvableCountProvableSumIndexedTree(..) - ); - let staged = cost_return_on_error!( - &mut cost, - self.mirror_indexed_primary_entry( - path.clone(), - &parent_merk, - &indexed_element, - key, - old_indexed_state, - tx.as_ref(), - &batch, - grove_version, - ) - ); - // The two seeds are not interchangeable: propagation rebuilds a - // single-axis element from `deferred_secondary` and a PCPSIT - // from `deferred_axes`, and seeding the wrong one leaves the - // other set for a later iteration that has no indexed element - // to apply it to. - if is_multi_axis { - (None, Some(staged)) - } else { - ( - staged.first().map(|(_, hash, key)| (*hash, key.clone())), - None, - ) - } - } else { - (None, None) - }; - let (initial_deferred_secondary, initial_deferred_axes) = initial_deferred; + // 5. Mirror the moved entry into the primary's axis secondaries, + // then propagate upward seeded with the mirror's per-axis roots + // so the indexed element one level up is rebuilt over the NEW + // secondary state rather than re-read from stale root keys. + let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( + &mut cost, + self.mirror_indexed_entry_and_seed( + path.clone(), + &parent_merk, + key, + old_indexed_state, + tx.as_ref(), + &batch, + grove_version, + ) + ); let mut merk_cache: HashMap, Merk> = HashMap::new(); diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index 8e2cbe347..d2a03713d 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -550,6 +550,104 @@ mod tests { ); } + /// Every direct non-Merk append API must refresh the canonical row of + /// the entry it rewrites. + /// + /// These four share a shape: they write the updated element straight + /// into the primary Merk and only then start propagating, so the + /// propagation walk — which mirrors entries it discovers as it climbs + /// — never sees the entry that moved. An append leaves `(count, sum)` + /// untouched (a non-Merk child contributes a constant count of `1`), + /// so under the old aggregate-only rows this was genuinely a no-op. + /// Under canonical rows it is not: the append rewrites the entry's + /// non-Merk root, and therefore its commitment. + /// + /// Each API gets its own case rather than one shared loop, because + /// each has its own copy of the write-then-propagate sequence and a + /// fix to one does not imply the others. + #[test] + fn every_non_merk_append_refreshes_the_row_it_rewrites() { + let grove_version = GroveVersion::latest(); + + let pcit_with_child = |child: Element, key: &[u8]| { + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("create PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + key, + child, + None, + grove_version, + ) + .unwrap() + .expect("non-Merk child"); + db + }; + + // MMR. + let db = pcit_with_child(Element::empty_mmr_tree(), b"mmr"); + db.mmr_tree_append( + [TEST_LEAF, b"cidx"].as_ref(), + b"mmr", + b"leaf".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("mmr append"); + let issues = db.verify_grovedb(None, true, false, grove_version).unwrap(); + assert!( + issues.is_empty(), + "mmr_tree_append left the row stale: {issues:?}" + ); + + // Bulk-append. + let db = pcit_with_child( + Element::empty_bulk_append_tree(4).expect("bulk tree"), + b"bulk", + ); + db.bulk_append( + [TEST_LEAF, b"cidx"].as_ref(), + b"bulk", + b"v".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + let issues = db.verify_grovedb(None, true, false, grove_version).unwrap(); + assert!( + issues.is_empty(), + "bulk_append left the row stale: {issues:?}" + ); + + // Dense. + let db = pcit_with_child(Element::empty_dense_tree(4), b"dense"); + db.dense_tree_insert( + [TEST_LEAF, b"cidx"].as_ref(), + b"dense", + b"v".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("dense insert"); + let issues = db.verify_grovedb(None, true, false, grove_version).unwrap(); + assert!( + issues.is_empty(), + "dense_tree_insert left the row stale: {issues:?}" + ); + } + /// Ordinary user references keep their own semantics. The one-hop /// immediate-node rule is dedicated indexed-tree behaviour, so it must /// not leak into how a normal reference elsewhere in the grove is From 70ab3b3347a757f4231aeee4e81e6ea6e846613a Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 04:28:04 +0700 Subject: [PATCH 04/18] feat: resolved values in indexed reads, via path-proof-free target chains (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the genuinely better parts of the alternative implementation in PR #816 while keeping this PR's proof-size property, and drops the merk wire-format change that #816 avoided. **Phase 3 is no longer deferred.** Every non-aggregate indexed read now returns `IndexedAxisEntry { ordering_value, primary_key, value }`, so a top-k result carries values rather than pointers — no follow-up `db.get` per row, and no extra inclusion proof per row for a verified read. A reference-shaped primary resolves to its TERMINAL, exactly as `db.get` on that key would, while the row stays BOUND to the immediate primary node so the mirror's invariant remains local. I previously judged this migration infeasible at ~500 call sites. #816 showed the way with a `PartialEq<(T, Vec)>` shim; the real cost was ~25 compile errors. The shim is documented as ignoring `value` — it answers "is this row in the right place", not "does it carry the right value" — and `key_pair()` is there for callers that genuinely only rank. Assertions that should check resolved values now do so explicitly, including one that previously could not tell a stale row from a fresh one at a fixed avg sort key. **Target chains replace the proof node variant.** Each returned row carries a chain of `(bytes, IndexedTargetCommitment)` entries — the immediate primary, then any reference hops to the terminal. The commitment enum (`Simple` / `Layered` / `IndexedSingle` / `IndexedMulti` / `Reference`) is #816's idea and it is the right one: it covers every target shape, including the nested indexed trees this PR previously refused with `NotSupported`. Unlike #816, a chain carries NO per-row path proofs. It authenticates itself from the row's own committed hash: each entry's commitment is rebuilt from its bytes plus the next entry's, and the head's is what the row binds. That is the same trust model shipped `KVRefValueHash*` proofs already use — they bind a reference's committed target hash to the returned value without separately proving the target's path inclusion — so a chain is neither weaker nor stronger than reading the same reference through an ordinary proof. Measured on a 32-entry PCIT with tree-shaped children, marginal proof cost per returned row is 83 bytes. Re-proving each primary from the grove root instead costs ~618 bytes/row and makes a k=16 proof 5.5x larger. A regression test pins the per-row figure. Because chains carry the layered commitment, the new merk `Node` variant this PR added is no longer needed: `grovedb-query` encoding, `proofs/tree.rs`, the merk verifiers and the chunk/branch matches are all reverted to develop. The proof wire change is now confined to the unshipped indexed-axis envelope. Full workspace suite passes (43 targets, 2736 grovedb tests); clippy three warnings below the pre-change baseline. --- docs/book/src/count-indexed-tree.md | 54 ++- grovedb-query/src/proofs/encoding.rs | 138 ------- grovedb-query/src/proofs/mod.rs | 44 --- .../src/estimated_costs/average_case_costs.rs | 6 - grovedb/src/lib.rs | 2 + grovedb/src/operations/indexed_tree.rs | 186 ++++++--- .../operations/proof/indexed_axis/envelope.rs | 95 ++++- .../operations/proof/indexed_axis/generate.rs | 246 +++++++++--- .../src/operations/proof/indexed_axis/mod.rs | 4 +- .../indexed_axis/reference_resolution.rs | 360 ------------------ .../proof/indexed_axis/target_chain.rs | 354 +++++++++++++++++ .../operations/proof/indexed_axis/verify.rs | 192 +++++----- grovedb/src/operations/proof/mod.rs | 21 +- grovedb/src/operations/proof/verify.rs | 15 +- grovedb/src/query_result_type.rs | 60 +++ grovedb/src/tests/axis_descent_proof_tests.rs | 4 +- .../tests/batch_indexed_fresh_create_tests.rs | 8 +- .../tests/batch_indexed_multi_axis_tests.rs | 37 +- grovedb/src/tests/coverage_round7_tests.rs | 12 +- .../indexed_axis_nested_and_bounds_tests.rs | 20 +- .../tests/indexed_axis_offset_proof_tests.rs | 4 +- .../indexed_axis_paginated_cost_tests.rs | 2 +- grovedb/src/tests/indexed_axis_proof_tests.rs | 54 ++- .../src/tests/indexed_reference_row_tests.rs | 156 ++++++++ .../indexed_tree_security_regression_tests.rs | 2 +- grovedb/src/tests/mod.rs | 1 + grovedb/src/tests/proof_size_measurement.rs | 94 +++++ .../tests/provable_count_sum_tree_tests.rs | 1 - merk/src/merk/chunks.rs | 3 - merk/src/proofs/branch/mod.rs | 6 +- merk/src/proofs/query/count_offset/verify.rs | 43 +-- merk/src/proofs/query/verify.rs | 25 -- merk/src/proofs/tree.rs | 36 +- 33 files changed, 1334 insertions(+), 951 deletions(-) delete mode 100644 grovedb/src/operations/proof/indexed_axis/reference_resolution.rs create mode 100644 grovedb/src/operations/proof/indexed_axis/target_chain.rs create mode 100644 grovedb/src/tests/proof_size_measurement.rs diff --git a/docs/book/src/count-indexed-tree.md b/docs/book/src/count-indexed-tree.md index e7ee5ff38..d3944ce32 100644 --- a/docs/book/src/count-indexed-tree.md +++ b/docs/book/src/count-indexed-tree.md @@ -180,9 +180,16 @@ combine_hash(H(reference bytes), primary_node_committed_value_hash) Three consequences worth stating plainly: -- **Reads and proofs return the primary value.** The reference is - resolved and authenticated as part of the axis proof, so a top-k result - carries the values, not just pointers to them. +- **Reads and proofs return the primary value.** Every non-aggregate + indexed read returns `IndexedAxisEntry { ordering_value, primary_key, + value }`, so a top-k result carries the values rather than pointers to + them — no follow-up `db.get` per row, and no extra inclusion proof per + row for a verified read. If the primary entry is itself a reference, + `value` is its TERMINAL, exactly as `db.get` on that key would give + you. + + Callers that genuinely only rank (leaderboards, ranking views) can drop + the value with `IndexedAxisEntry::key_pair`. - **The binding is to the IMMEDIATE primary node**, not to a terminal reached by following a chain. That keeps the invariant local: the only thing that can staleness a row is a write to the primary entry itself, @@ -472,7 +479,7 @@ Merk is not touched. The verifier receives the primary's root hash plus a ```rust // Shipped API on `GroveDb`: -let entries: Vec<(u64, Vec)> = db +let entries: Vec> = db .indexed_count_top_k(path, k, /* descending: */ true, transaction, grove_version)? .expect("top-k"); @@ -481,10 +488,11 @@ let proof_bytes = db .prove_indexed_count_top_k(path, k, /* descending: */ true, transaction, grove_version)? .expect("prove"); let result = GroveDb::verify_indexed_count_top_k(&proof_bytes, &path, k)?; -// result.entries: Vec<(u64, Vec)>, result.root_hash: [u8; 32] +// result.entries: Vec>, result.root_hash: [u8; 32] ``` -The query returns `(count, key)` pairs. +The query returns `IndexedAxisEntry` rows — the count, the primary key, +and the resolved primary value. Internally: @@ -494,22 +502,32 @@ Internally: 3. Run a **descending range query** with `limit = k` over the full secondary keyspace. This yields the k highest-count entries, with a standard Merk range proof. -4. Resolve each returned row through its reference and emit a - reference-aware proof node carrying the primary value. This costs no - extra Merk inclusion proof: a row IS a reference to its primary - entry, and the resolved value is folded into the row's own committed - hash (see [Secondary rows](#secondary-rows)). - -The verifier authenticates the resolved value, and with it the row's -reference path, hop budget and carried sum, by rebuilding the canonical -row the primary value implies and comparing it against what the proof -committed. A row filed under one key whose reference points at another -cannot verify. +4. Attach one **target chain** per returned row: the immediate primary + entry, then any ordinary reference hops through to the terminal. Each + chain entry carries its serialized bytes plus the rule that turns them + into a commitment (`Simple`, `Layered`, `IndexedSingle`, + `IndexedMulti`, `Reference`). + +A chain carries **no per-row path proofs**. It authenticates itself from +the row's own committed hash: each entry's commitment is rebuilt from its +bytes plus the next entry's, and the head's is what the row binds — and +the row is bound into the secondary root, the indexed element, and the +grove root. That is the same trust model shipped GroveDB reference proofs +already use, so a chain is neither weaker nor stronger than reading the +same reference through an ordinary proof. The practical effect is that a +top-k result costs roughly one value plus one hash per row, instead of +`k` inclusion proofs. + +The verifier also rebuilds the canonical row that the resolved primary +value implies, and compares it against what the proof carried. That one +comparison covers the ordering prefix, the primary-key suffix, the +reference path, the hop budget and the carried sum — so a row filed under +one key whose reference points at another cannot verify. ### Range by count ```rust -let entries: Vec<(u64, Vec)> = db +let entries: Vec> = db .indexed_count_range( path, min, // u64, inclusive diff --git a/grovedb-query/src/proofs/encoding.rs b/grovedb-query/src/proofs/encoding.rs index 9f8178dc0..a864ca369 100644 --- a/grovedb-query/src/proofs/encoding.rs +++ b/grovedb-query/src/proofs/encoding.rs @@ -540,35 +540,6 @@ impl Encode for Op { sum.encode_into(dest)?; } } - Op::Push(Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - count, - sum, - target_child_hash, - )) => { - let key_len = key_len_u8(key)?; - if value.len() < 65536 { - dest.write_all(&[0x4e, key_len])?; - dest.write_all(key)?; - (value.len() as u16).encode_into(dest)?; - dest.write_all(value)?; - dest.write_all(value_hash)?; - count.encode_into(dest)?; - sum.encode_into(dest)?; - dest.write_all(target_child_hash)?; - } else { - dest.write_all(&[0x4f, key_len])?; - dest.write_all(key)?; - (value.len() as u32).encode_into(dest)?; - dest.write_all(value)?; - dest.write_all(value_hash)?; - count.encode_into(dest)?; - sum.encode_into(dest)?; - dest.write_all(target_child_hash)?; - } - } Op::Push(Node::KVDigestCountSum(key, value_hash, count, sum)) => { let key_len = key_len_u8(key)?; @@ -638,35 +609,6 @@ impl Encode for Op { sum.encode_into(dest)?; } } - Op::PushInverted(Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - count, - sum, - target_child_hash, - )) => { - let key_len = key_len_u8(key)?; - if value.len() < 65536 { - dest.write_all(&[0x50, key_len])?; - dest.write_all(key)?; - (value.len() as u16).encode_into(dest)?; - dest.write_all(value)?; - dest.write_all(value_hash)?; - count.encode_into(dest)?; - sum.encode_into(dest)?; - dest.write_all(target_child_hash)?; - } else { - dest.write_all(&[0x51, key_len])?; - dest.write_all(key)?; - (value.len() as u32).encode_into(dest)?; - dest.write_all(value)?; - dest.write_all(value_hash)?; - count.encode_into(dest)?; - sum.encode_into(dest)?; - dest.write_all(target_child_hash)?; - } - } Op::PushInverted(Node::KVDigestCountSum(key, value_hash, count, sum)) => { let key_len = key_len_u8(key)?; @@ -848,23 +790,6 @@ impl Encode for Op { + count.encoding_length()? + sum.encoding_length()? } - Op::Push(Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - _, - count, - sum, - _, - )) => { - let header = if value.len() < 65536 { 4 } else { 6 }; - header - + key.len() - + value.len() - + HASH_LENGTH - + count.encoding_length()? - + sum.encoding_length()? - + HASH_LENGTH - } Op::Push(Node::KVDigestCountSum(key, _, count, sum)) => { 2 + key.len() + HASH_LENGTH + count.encoding_length()? + sum.encoding_length()? } @@ -892,23 +817,6 @@ impl Encode for Op { + count.encoding_length()? + sum.encoding_length()? } - Op::PushInverted(Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - _, - count, - sum, - _, - )) => { - let header = if value.len() < 65536 { 4 } else { 6 }; - header - + key.len() - + value.len() - + HASH_LENGTH - + count.encoding_length()? - + sum.encoding_length()? - + HASH_LENGTH - } Op::PushInverted(Node::KVDigestCountSum(key, _, count, sum)) => { 2 + key.len() + HASH_LENGTH + count.encoding_length()? + sum.encoding_length()? } @@ -1947,52 +1855,6 @@ impl Decode for Op { sum, )) } - // Resolved reference with a layered target. Same layout as - // `KVRefValueHashCountSum` (0x43/0x44, 0x4a/0x4b) plus a - // trailing 32-byte target child hash. - 0x4e | 0x4f | 0x50 | 0x51 => { - let inverted = matches!(variant, 0x50 | 0x51); - let long_value = matches!(variant, 0x4f | 0x51); - let key_len: u8 = Decode::decode(&mut input)?; - let mut key = vec![0; key_len as usize]; - input.read_exact(key.as_mut_slice())?; - - let value_len: u32 = if long_value { - let len: u32 = Decode::decode(&mut input)?; - if len > MAX_VALUE_LEN { - return Err(ed::Error::UnexpectedByte(variant)); - } - len - } else { - let len: u16 = Decode::decode(&mut input)?; - len as u32 - }; - let mut value = vec![0; value_len as usize]; - input.read_exact(value.as_mut_slice())?; - - let mut value_hash = [0; HASH_LENGTH]; - input.read_exact(&mut value_hash)?; - - let count: u64 = Decode::decode(&mut input)?; - let sum: i64 = Decode::decode(&mut input)?; - - let mut target_child_hash = [0; HASH_LENGTH]; - input.read_exact(&mut target_child_hash)?; - - let node = Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - count, - sum, - target_child_hash, - ); - if inverted { - Self::PushInverted(node) - } else { - Self::Push(node) - } - } 0x10 => Self::Parent, 0x11 => Self::Child, diff --git a/grovedb-query/src/proofs/mod.rs b/grovedb-query/src/proofs/mod.rs index fb70ee206..6898b1361 100644 --- a/grovedb-query/src/proofs/mod.rs +++ b/grovedb-query/src/proofs/mod.rs @@ -245,33 +245,6 @@ pub enum Node { /// /// Contains: `(kv_hash, left_child_hash, right_child_hash, count, sum)` HashWithCountAndSum(CryptoHash, CryptoHash, CryptoHash, u64, i64), - - /// A resolved reference whose TARGET is itself layered — a subtree, an - /// indexed element, or another reference — so the target's committed - /// value hash is a combined hash rather than `H(target_value)`. - /// - /// `KVRefValueHashCountSum` cannot express these: it reconstructs the - /// row's commitment as - /// `combine_hash(reference_element_hash, H(referenced_value))`, and no - /// choice of `referenced_value` makes `H(referenced_value)` equal a - /// combined hash. This variant carries the missing half — the target's - /// own child/next-hop commitment — so the verifier can rebuild the - /// full chain: - /// - /// ```text - /// target_committed = combine_hash(H(referenced_value), target_child_hash) - /// row_committed = combine_hash(reference_element_hash, target_committed) - /// ``` - /// - /// Introduced for indexed-tree axis proofs, where the rows are - /// canonical references and the primary entries they point at are - /// routinely subtrees (a count-indexed tree indexes its children's - /// counts, so tree-shaped children are the normal case, not an edge - /// case). - /// - /// Contains: `(key, referenced_value, reference_element_hash, count, - /// sum, target_child_hash)` - KVRefValueHashCountSumWithTargetChildHash(Vec, Vec, CryptoHash, u64, i64, CryptoHash), } use std::fmt; @@ -401,23 +374,6 @@ impl fmt::Display for Node { count, sum ), - Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - count, - sum, - target_child_hash, - ) => format!( - "KVRefValueHashCountSumWithTargetChildHash({}, {}, HASH[{}], count={}, sum={}, \ - target_child=HASH[{}])", - hex_to_ascii(key), - hex_to_ascii(value), - hex::encode(value_hash), - count, - sum, - hex::encode(target_child_hash) - ), Node::KVDigestCountSum(key, value_hash, count, sum) => format!( "KVDigestCountSum({}, HASH[{}], count={}, sum={})", hex_to_ascii(key), diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 7f339078a..290fd7cb6 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -31,12 +31,6 @@ use crate::{ Element, ElementFlags, Error, GroveDb, }; -/// Upper bound on an indexed secondary row's value. The payload is fixed -/// per axis — a `SumItem` (count and sum axes) or an empty -/// `ItemWithSumItem` (avg) — and the largest of those serializes well under -/// this bound, which also leaves room for the feature type and flags byte. -pub const INDEXED_SECONDARY_MAX_VALUE_SIZE: u32 = 16; - impl GroveDb { /// Add average case for getting a merk tree pub fn add_average_case_get_merk_at_path<'db, S: Storage<'db>>( diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 77d478496..95dd7a0c7 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -259,6 +259,8 @@ pub use query::{ GroveTrunkQueryResult, LeafInfo, PathBranchChunkQuery, PathQuery, PathQueryShape, PathTrunkChunkQuery, SizedQuery, }; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use query_result_type::IndexedAxisEntry; #[cfg(feature = "minimal")] use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] diff --git a/grovedb/src/operations/indexed_tree.rs b/grovedb/src/operations/indexed_tree.rs index 9f8fa1412..44ed9d598 100644 --- a/grovedb/src/operations/indexed_tree.rs +++ b/grovedb/src/operations/indexed_tree.rs @@ -51,7 +51,10 @@ use grovedb_storage::{ }; use grovedb_version::version::GroveVersion; -use crate::{util::TxRef, Element, Error, GroveDb, Transaction, TransactionArg}; +use crate::{ + query_result_type::IndexedAxisEntry, util::TxRef, Element, Error, GroveDb, Transaction, + TransactionArg, +}; /// Per-axis Merk tree type to open the secondary with. /// @@ -242,31 +245,6 @@ pub(crate) fn decode_axis_row_reference<'a>( } } -/// The primary item key a canonical row points at, cross-checked against -/// the item key its own secondary-key suffix encodes. -/// -/// The suffix and the reference path are two independent encodings of the -/// same fact, and a proof or verification pass that trusts one without -/// the other accepts a row whose committed reference points somewhere the -/// sort position does not. Both the verifier and the proof generator go -/// through here so neither can drift into checking only one. -pub(crate) fn indexed_row_target_key<'a>( - row: &'a Element, - key_suffix_item_key: &[u8], - describe: &str, -) -> Result<(&'a [u8], i64), Error> { - let (target, sum) = decode_axis_row_reference(row, describe)?; - if target != key_suffix_item_key { - return Err(Error::CorruptedData(format!( - "{describe}: indexed secondary row references {} but its secondary-key suffix \ - encodes {}", - hex::encode(target), - hex::encode(key_suffix_item_key) - ))); - } - Ok((target, sum)) -} - /// Build the secondary key bytes for an entry at `item_key` under the /// given axis, given the relevant aggregate values: /// - count axis → `count_be(8) ‖ item_key` @@ -1508,7 +1486,7 @@ impl GroveDb { transaction: TransactionArg, grove_version: &GroveVersion, decode: impl Fn(&[u8]) -> Option<(T, Vec)>, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, { @@ -1522,10 +1500,15 @@ impl GroveDb { let secondary_merk = cost_return_on_error!( &mut cost, - self.open_validated_axis_secondary(path, axis, tx_ref, grove_version) + self.open_validated_axis_secondary(path.clone(), axis, tx_ref, grove_version) ); - collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode).add_cost(cost) + let rows = cost_return_on_error!( + &mut cost, + collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode) + ); + drop(secondary_merk); + resolve_axis_entries(self, path, rows, transaction, grove_version).add_cost(cost) } /// One implementation of the `indexed__top_k_paginated` shape. @@ -1564,7 +1547,12 @@ impl GroveDb { // skips zero entries, so `skipped = min(0, population) = 0` needs // no tree read. if offset == 0 { - return collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode) + let rows = cost_return_on_error!( + &mut cost, + collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode) + ); + drop(secondary_merk); + return resolve_axis_entries(self, path, rows, transaction, grove_version) .map_ok(|entries| IndexedTopKPage { entries, skipped: 0, @@ -1598,6 +1586,9 @@ impl GroveDb { }; let parent_prefix = RocksDbStorage::build_prefix(parent_path.clone()).unwrap_add_cost(&mut cost); + // Kept for resolving the page's primary values once the counted + // descent has produced its keys. + let path_for_resolution = path.clone(); let primary_prefix = RocksDbStorage::build_prefix(path).unwrap_add_cost(&mut cost); let secondary_prefix = RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) .unwrap_add_cost(&mut cost); @@ -1647,17 +1638,19 @@ impl GroveDb { grove_version ) ); - let mut entries = Vec::with_capacity(secondary_keys.len()); + let mut rows = Vec::with_capacity(secondary_keys.len()); for secondary_key in secondary_keys { match decode(&secondary_key) { - Some(decoded) => entries.push(decoded), + Some(decoded) => rows.push(decoded), None => { return Err(corrupted_secondary_key_error(axis, &secondary_key)) .wrap_with_cost(cost); } } } - Ok(IndexedTopKPage { entries, skipped }).wrap_with_cost(cost) + resolve_axis_entries(self, path_for_resolution, rows, transaction, grove_version) + .map_ok(|entries| IndexedTopKPage { entries, skipped }) + .add_cost(cost) } /// One implementation of the `indexed__range` shape. The @@ -1678,7 +1671,7 @@ impl GroveDb { transaction: TransactionArg, grove_version: &GroveVersion, decode: impl Fn(&[u8]) -> Option<(T, Vec)>, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, { @@ -1692,7 +1685,7 @@ impl GroveDb { let secondary_merk = cost_return_on_error!( &mut cost, - self.open_validated_axis_secondary(path, axis, tx_ref, grove_version) + self.open_validated_axis_secondary(path.clone(), axis, tx_ref, grove_version) ); let mut q = Query::new(); @@ -1718,8 +1711,10 @@ impl GroveDb { None => break, } } + drop(iter); + drop(secondary_merk); - Ok(results).wrap_with_cost(cost) + resolve_axis_entries(self, path, results, transaction, grove_version).add_cost(cost) } // ---- count axis ---- @@ -1748,7 +1743,7 @@ impl GroveDb { descending: bool, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -1822,7 +1817,7 @@ impl GroveDb { limit: u16, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -1966,7 +1961,7 @@ impl GroveDb { descending: bool, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -2030,7 +2025,7 @@ impl GroveDb { limit: u16, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -2292,7 +2287,7 @@ impl GroveDb { descending: bool, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -2365,7 +2360,7 @@ impl GroveDb { limit: u16, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult)>, Error> + ) -> CostResult>, Error> where B: AsRef<[u8]> + 'b, P: Into>, @@ -2831,7 +2826,7 @@ fn axis_sort_prefix_len(axis: IndexAxis) -> usize { /// surface storage corruption (rather than silently dropping the entry) /// when the secondary's keyspace contains a malformed entry. #[inline] -fn corrupted_secondary_key_error(axis: IndexAxis, secondary_key: &[u8]) -> Error { +pub(crate) fn corrupted_secondary_key_error(axis: IndexAxis, secondary_key: &[u8]) -> Error { Error::CorruptedData(format!( "secondary key in indexed-tree (axis {:?}) is shorter than {} bytes: {:?}", axis, @@ -2877,6 +2872,85 @@ fn collect_top_k_via_iterator<'db, S: StorageContext<'db>, T>( Ok(results).wrap_with_cost(cost) } +/// Turn decoded `(ordering_value, primary_key)` pairs into full entries by +/// reading each primary value. +/// +/// Resolution applies ORDINARY GroveDB reference semantics: a +/// reference-shaped primary entry resolves to its terminal, exactly as +/// `db.get` on that key would. That is deliberately not the rule a row is +/// *bound* by — a row commits its immediate primary node, which is what +/// keeps the mirror's invariant local — and the two stay consistent +/// because the immediate node's commitment transitively covers whatever it +/// pointed at when written. +/// +/// The primary Merk is opened once for the whole page rather than per row. +fn resolve_axis_entries<'b, B, T>( + db: &GroveDb, + indexed_path: SubtreePath<'b, B>, + rows: Vec<(T, Vec)>, + transaction: TransactionArg, + grove_version: &GroveVersion, +) -> CostResult>, Error> +where + B: AsRef<[u8]> + 'b, +{ + let mut cost = OperationCost::default(); + if rows.is_empty() { + return Ok(Vec::new()).wrap_with_cost(cost); + } + + let tx = TxRef::new(&db.db, transaction); + let primary_merk = cost_return_on_error!( + &mut cost, + db.open_transactional_merk_at_path(indexed_path.clone(), tx.as_ref(), None, grove_version) + ); + + let mut entries = Vec::with_capacity(rows.len()); + for (ordering_value, primary_key) in rows { + let element = cost_return_on_error!( + &mut cost, + Element::get(&primary_merk, &primary_key, true, grove_version).map_err(|e| { + Error::CorruptedData(format!( + "indexed axis read: primary entry {} named by a secondary row is missing: {e}", + hex::encode(&primary_key) + )) + }) + ); + let value = match element.underlying() { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { + let mut qualified = indexed_path.to_vec(); + qualified.push(primary_key.clone()); + let absolute = match crate::reference_path::path_from_reference_path_type( + reference_path.clone(), + &qualified, + Some(primary_key.as_slice()), + ) { + Ok(p) => p, + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + }; + cost_return_on_error!( + &mut cost, + db.follow_reference( + absolute.as_slice().into(), + true, + transaction, + grove_version + ) + ) + } + _ => element, + }; + entries.push(IndexedAxisEntry { + ordering_value, + primary_key, + value, + }); + } + + Ok(entries).wrap_with_cost(cost) +} + /// Strict provable-count read of an aggregate. The counted skip only ever /// runs against axis secondaries, whose tree types (`ProvableCountTree` / /// `ProvableCountProvableSumTree`) bind a provable count into every node; @@ -2901,8 +2975,9 @@ fn provable_count_from_aggregate(aggregate: AggregateData) -> Result /// One page of an `indexed__top_k_paginated` read. #[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexedTopKPage { - /// Page entries, `(axis_value, original_key)`, in directional order. - pub entries: Vec<(T, Vec)>, + /// Page entries in directional order, each carrying its resolved + /// primary value. + pub entries: Vec>, /// How many entries the offset actually skipped: /// `min(offset, population)`. When the offset runs past the end this /// reports the secondary's true population instead of echoing the @@ -3285,6 +3360,10 @@ impl GroveDb { /// change from the replaced code). Kept as the measurement baseline /// for `indexed_axis_paginated_cost_tests::measure_paginated_costs`; /// compiled only for tests, never reachable in production builds. + /// + /// Returns bare `(count, key)` pairs, NOT resolved entries: it exists + /// to measure what the SECONDARY traversal costs, and resolving + /// primary values would fold an unrelated cost into the baseline. pub(crate) fn legacy_linear_indexed_count_top_k_paginated<'b, B, P>( &self, path: P, @@ -3369,8 +3448,7 @@ mod axis_row_reference_tests { use grovedb_version::version::GroveVersion; use super::{ - axis_payload_sum, axis_row_reference, decode_axis_row_reference, indexed_row_target_key, - INDEXED_SECONDARY_MAX_HOP, + axis_payload_sum, axis_row_reference, decode_axis_row_reference, INDEXED_SECONDARY_MAX_HOP, }; use crate::Element; @@ -3446,17 +3524,9 @@ mod axis_row_reference_tests { decode_axis_row_reference(&row, "test").unwrap(), (b"target".as_slice(), 42) ); - assert_eq!( - indexed_row_target_key(&row, b"target", "test").unwrap(), - (b"target".as_slice(), 42) - ); - - // Suffix disagreement: the row's reference and its sort-key suffix - // are two independent encodings of the same fact, and trusting one - // without the other accepts a row whose commitment points somewhere - // its sort position does not. - indexed_row_target_key(&row, b"other", "test") - .expect_err("reference target must agree with the secondary-key suffix"); + // Suffix agreement is now enforced by the verifier rebuilding the + // canonical row from the authenticated primary value, so it has no + // separate helper to unit-test here. // Legacy placeholder rows must be rejected outright. for legacy in [ diff --git a/grovedb/src/operations/proof/indexed_axis/envelope.rs b/grovedb/src/operations/proof/indexed_axis/envelope.rs index 4f420b1b7..f7cc5f828 100644 --- a/grovedb/src/operations/proof/indexed_axis/envelope.rs +++ b/grovedb/src/operations/proof/indexed_axis/envelope.rs @@ -9,6 +9,8 @@ use bincode::{Decode, Encode}; use grovedb_element::indexed::IndexAxis; use grovedb_merk::tree::CryptoHash; +use crate::IndexedAxisEntry; + /// Per-ancestor attestation for chaining the cidx/psit/pcpsit layer /// composition during verification. /// @@ -41,6 +43,76 @@ pub enum AncestorAttestation { MultiAxis(Vec<(u8, [u8; 32])>), } +/// How a node's serialized element bytes compose into the value hash its +/// parent Merk committed. +/// +/// This is what makes a resolved indexed row shape-complete: the element +/// bytes alone determine the commitment only for item-like values, and +/// every other shape folds in something the bytes do not carry. +#[derive(Encode, Decode, Debug, Clone, PartialEq, Eq)] +pub enum IndexedTargetCommitment { + /// Item-like value, committed as `H(value)`. + Simple, + /// Merk-backed or non-Merk tree, committed as + /// `combine_hash(H(value), child_root_hash)`. + Layered([u8; 32]), + /// PCIT / PSIT, committed as + /// `combine_hash_three(H(value), primary_root, secondary_root)`. + IndexedSingle { + /// Root hash of the indexed tree's primary Merk. + primary_root_hash: [u8; 32], + /// Root hash of its only secondary Merk. + secondary_root_hash: [u8; 32], + }, + /// PCPSIT, committed as + /// `combine_hash_three(H(value), primary_root, axes_digest(axes))`. + IndexedMulti { + /// Root hash of the indexed tree's primary Merk. + primary_root_hash: [u8; 32], + /// Canonical `(axis_tag, secondary_root_hash)` list, tag-sorted. + axes: Vec<(u8, [u8; 32])>, + }, + /// Reference node, committed as + /// `combine_hash(H(value), next_node_commitment)` — the next hop's + /// commitment is reconstructed from the following chain entry rather + /// than carried, which is what keeps a chain self-authenticating. + Reference, +} + +/// One node of a resolved target chain: its serialized element bytes and +/// the shape rule that turns them into a commitment. +#[derive(Encode, Decode, Debug, Clone, PartialEq, Eq)] +pub struct IndexedTargetNode { + /// The node's serialized element bytes. + pub value: Vec, + /// How those bytes compose into the value hash its parent committed. + pub commitment: IndexedTargetCommitment, +} + +/// The resolved target of one secondary row: the immediate primary node, +/// followed by any ordinary reference hops through to the terminal. +/// +/// **No per-node path proofs.** A chain authenticates itself from the +/// row's own committed value hash: each entry's commitment is +/// reconstructed from its bytes plus the NEXT entry's commitment, and the +/// head's commitment is what the row binds. Since the row's hash is bound +/// into the secondary root — and that into the indexed element, and that +/// to the grove root — substituting any value in the chain breaks the +/// root. +/// +/// That is the same trust model shipped GroveDB reference proofs already +/// use (`KVRefValueHash*` binds a reference's committed target hash to the +/// returned value without separately proving the target's path +/// inclusion), so a chain is neither weaker nor stronger than reading the +/// same reference through an ordinary proof. It is what lets a top-k +/// result carry `k` values for a per-row cost of the value plus a hash, +/// instead of `k` inclusion proofs. +#[derive(Encode, Decode, Debug, Clone, PartialEq, Eq)] +pub struct IndexedTargetChain { + /// Immediate primary node first, terminal last. Never empty. + pub nodes: Vec, +} + /// Wire-format envelope for a range / top-k / arbitrary-query proof /// over an indexed-tree's per-axis secondary index. #[derive(Encode, Decode, Debug)] @@ -79,6 +151,9 @@ pub struct IndexedAxisRangeProof { pub target_is_pcpsit: bool, /// Encoded Merk range proof for the per-axis secondary. pub secondary_proof: Vec, + /// One resolved-target chain per returned secondary row, in the + /// secondary proof's result order. + pub target_chains: Vec, /// Echoed query limit (preserves `None`-vs-`Some(0)` semantics). pub requested_limit: Option, /// Echoed iteration direction. `false` = ascending, `true` = @@ -114,6 +189,8 @@ pub struct IndexedAxisPaginatedProof { /// `prove_count_offset_on_range`-produced `Vec` stream (every /// axis's secondary carries a provable count). pub secondary_proof: Vec, + /// Same as [`IndexedAxisRangeProof::target_chains`]. + pub target_chains: Vec, /// Echoed pagination parameters. pub requested_k: u16, /// Echoed offset. @@ -165,12 +242,12 @@ pub struct IndexedAxisAggregateProof { /// [`IndexedAxisQueryResult::entries`]. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AxisEntries { - /// Count-axis entries: `(count_value, original_key)`. - Count(Vec<(u64, Vec)>), - /// Sum-axis entries: `(sum_value, original_key)`. - Sum(Vec<(i64, Vec)>), - /// Avg-axis entries: `(avg_fixed_point_i128, original_key)`. - Avg(Vec<(i128, Vec)>), + /// Count-axis entries, ordered by count. + Count(Vec>), + /// Sum-axis entries, ordered by sum. + Sum(Vec>), + /// Avg-axis entries, ordered by fixed-point average. + Avg(Vec>), } impl AxisEntries { @@ -197,9 +274,9 @@ impl AxisEntries { /// item of a `k = 1` page, used by rank verification. pub fn first_original_key(&self) -> Option<&[u8]> { match self { - AxisEntries::Count(entries) => entries.first().map(|(_, key)| key.as_slice()), - AxisEntries::Sum(entries) => entries.first().map(|(_, key)| key.as_slice()), - AxisEntries::Avg(entries) => entries.first().map(|(_, key)| key.as_slice()), + AxisEntries::Count(entries) => entries.first().map(|e| e.primary_key.as_slice()), + AxisEntries::Sum(entries) => entries.first().map(|e| e.primary_key.as_slice()), + AxisEntries::Avg(entries) => entries.first().map(|e| e.primary_key.as_slice()), } } diff --git a/grovedb/src/operations/proof/indexed_axis/generate.rs b/grovedb/src/operations/proof/indexed_axis/generate.rs index 5bb3d9e87..615d76d8c 100644 --- a/grovedb/src/operations/proof/indexed_axis/generate.rs +++ b/grovedb/src/operations/proof/indexed_axis/generate.rs @@ -11,6 +11,7 @@ use grovedb_costs::{ use grovedb_element::indexed::IndexAxis; use grovedb_merk::{ element::get::ElementFetchFromStorageExtensions, + proofs::query::{verify_count_offset_on_range_proof, QueryProofVerify}, proofs::{encode_into, query::QueryItem as MerkQueryItemForRange, Query as MerkQuery}, }; use grovedb_path::{SubtreePath, SubtreePathBuilder}; @@ -27,6 +28,143 @@ use super::{ }; use crate::operations::proof::AxisDescentProof; +/// Replay a just-built secondary range proof to learn the exact rows the +/// verifier will see, then build one resolved-target chain per row. +/// +/// Replaying rather than reusing the prover's own bookkeeping is +/// deliberate: chains are matched to rows positionally at verify time, so +/// the two lists must be derived from the same source of truth. Anything +/// that changes which rows a proof yields — a limit boundary, a direction +/// flip — then changes both together or neither. +#[allow(clippy::too_many_arguments)] +fn build_row_target_chains<'db>( + grovedb: &'db GroveDb, + axis: IndexAxis, + secondary_proof: &[u8], + secondary_query: MerkQuery, + limit: Option, + indexed_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult, Error> { + let mut cost = OperationCost::default(); + if secondary_proof.is_empty() { + return Ok(Vec::new()).wrap_with_cost(cost); + } + let left_to_right = secondary_query.left_to_right; + let (_, sec_result) = cost_return_on_error!( + &mut cost, + secondary_query + .execute_proof(secondary_proof, limit, left_to_right, 0) + .map_err(|e| Error::CorruptedData(format!( + "indexed-axis proof: replaying the secondary proof for row order: {e}" + ))) + ); + let keys: Vec> = sec_result + .result_set + .into_iter() + .map(|proved| proved.key) + .collect(); + build_chains_for_keys( + grovedb, + axis, + &keys, + indexed_path, + transaction, + batch, + grove_version, + ) + .add_cost(cost) +} + +/// Count-offset twin of [`build_row_target_chains`]. +#[allow(clippy::too_many_arguments)] +fn build_paginated_target_chains<'db>( + grovedb: &'db GroveDb, + axis: IndexAxis, + secondary_proof: &[u8], + offset: u64, + k: u16, + descending: bool, + indexed_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult, Error> { + let mut cost = OperationCost::default(); + if secondary_proof.is_empty() { + return Ok(Vec::new()).wrap_with_cost(cost); + } + let inner_range = MerkQueryItemForRange::RangeFull(std::ops::RangeFull); + let result = cost_return_on_error!( + &mut cost, + verify_count_offset_on_range_proof( + secondary_proof, + &inner_range, + offset, + Some(k as u64), + !descending, + ) + .map_err(|e| Error::CorruptedData(format!( + "indexed-axis proof: replaying the paginated secondary proof for row order: {e}" + ))) + ); + let keys: Vec> = result + .returned_items + .into_iter() + .map(|item| item.key) + .collect(); + build_chains_for_keys( + grovedb, + axis, + &keys, + indexed_path, + transaction, + batch, + grove_version, + ) + .add_cost(cost) +} + +/// Turn secondary keys into per-row chains, decoding each key's primary +/// suffix on the way. +fn build_chains_for_keys<'db>( + grovedb: &'db GroveDb, + axis: IndexAxis, + secondary_keys: &[Vec], + indexed_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult, Error> { + let mut cost = OperationCost::default(); + let sort_len = crate::operations::indexed_tree::axis_sort_key_len(axis); + let mut chains = Vec::with_capacity(secondary_keys.len()); + for secondary_key in secondary_keys { + if secondary_key.len() < sort_len { + return Err( + crate::operations::indexed_tree::corrupted_secondary_key_error(axis, secondary_key), + ) + .wrap_with_cost(cost); + } + let primary_key = &secondary_key[sort_len..]; + let chain = cost_return_on_error!( + &mut cost, + super::target_chain::build_target_chain( + grovedb, + indexed_path, + primary_key, + transaction, + batch, + grove_version, + ) + ); + chains.push(chain); + } + Ok(chains).wrap_with_cost(cost) +} + /// Build the per-ancestor attestation list for a path of length N: the /// list has length N-1 (one entry per intermediate layer). For each /// intermediate layer, open the parent merk and inspect the element @@ -856,7 +994,11 @@ impl GroveDb { ); let descending = !secondary_query.left_to_right; let requested_limit = limit; - let mut sec_result = cost_return_on_error!( + // Kept for the row-order replay below: the prover re-executes its + // own proof to learn exactly which rows, in which order, the + // verifier will see — so chains cannot drift out of alignment. + let secondary_query_for_rows = secondary_query.clone(); + let sec_result = cost_return_on_error!( &mut cost, secondary_merk .prove_without_encoding(secondary_query, limit, grove_version) @@ -864,24 +1006,25 @@ impl GroveDb { "indexed-axis range proof: secondary range proof: {e}" ))) ); - // Resolve canonical rows into the primary values they point at, - // before encoding, so the verified result carries the referenced - // Element rather than the reference. - cost_return_on_error!( + let mut serialized_secondary = Vec::with_capacity(128); + encode_into(sec_result.proof.iter(), &mut serialized_secondary); + // One resolved-target chain per returned row, in result order. + // The rows themselves stay raw references in the merk proof; the + // chain is what carries (and binds) the value they point at. + let target_chains = cost_return_on_error!( &mut cost, - super::reference_resolution::resolve_axis_reference_nodes( - sec_result.proof.iter_mut(), - axis, - &primary_merk, + build_row_target_chains( self, + axis, + &serialized_secondary, + secondary_query_for_rows, + requested_limit, &path_keys, transaction, batch, grove_version, ) ); - let mut serialized_secondary = Vec::with_capacity(128); - encode_into(sec_result.proof.iter(), &mut serialized_secondary); Ok(IndexedAxisRangeProof { axis_tag: axis.tag(), @@ -891,6 +1034,7 @@ impl GroveDb { other_axes_root_hashes, target_is_pcpsit, secondary_proof: serialized_secondary, + target_chains, requested_limit, descending, }) @@ -1011,7 +1155,7 @@ impl GroveDb { .wrap_with_cost(cost); } let inner_range = MerkQueryItemForRange::RangeFull(std::ops::RangeFull); - let mut prove_result = cost_return_on_error!( + let prove_result = cost_return_on_error!( &mut cost, secondary_merk .prove_count_offset_on_range( @@ -1025,25 +1169,23 @@ impl GroveDb { "indexed-axis paginated proof: secondary count-offset proof: {e}" ))) ); - // Resolve canonical rows into the primary values they point at, - // BEFORE encoding — the count-offset path used to return here - // without a reference post-pass, which is why it rejected - // reference rows outright. - cost_return_on_error!( + let mut serialized = Vec::with_capacity(128); + encode_into(prove_result.ops.iter(), &mut serialized); + let target_chains = cost_return_on_error!( &mut cost, - super::reference_resolution::resolve_axis_reference_nodes( - prove_result.ops.iter_mut(), - axis, - &primary_merk, + build_paginated_target_chains( self, + axis, + &serialized, + offset, + k, + descending, &path_keys, transaction, batch, grove_version, ) ); - let mut serialized = Vec::with_capacity(128); - encode_into(prove_result.ops.iter(), &mut serialized); Ok(IndexedAxisPaginatedProof { axis_tag: axis.tag(), @@ -1053,6 +1195,7 @@ impl GroveDb { other_axes_root_hashes, target_is_pcpsit, secondary_proof: serialized, + target_chains, requested_k: k, requested_offset: offset, descending, @@ -1292,13 +1435,13 @@ impl GroveDb { grove_version, ) ); + let mut target_chains: Vec = Vec::new(); let secondary_proof = match &axis_query.traversal { AxisTraversal::RankedPage { k, offset } => { - cost_return_on_error_no_add!( + let (bytes, chains) = cost_return_on_error_no_add!( cost, build_paginated_secondary_proof( &secondary_merk, - &primary_merk, self, &path_keys, transaction, @@ -1310,17 +1453,18 @@ impl GroveDb { grove_version, &mut cost, ) - ) + ); + target_chains = chains; + bytes } AxisTraversal::RankOfKey { .. } => { // rank computed above; the rank proof IS the paginated // proof at (offset = rank, k = 1). let rank_offset = rank.expect("set above for RankOfKey"); - cost_return_on_error_no_add!( + let (bytes, chains) = cost_return_on_error_no_add!( cost, build_paginated_secondary_proof( &secondary_merk, - &primary_merk, self, &path_keys, transaction, @@ -1332,7 +1476,9 @@ impl GroveDb { grove_version, &mut cost, ) - ) + ); + target_chains = chains; + bytes } AxisTraversal::Bounded { limit, .. } => { // An empty secondary cannot carry a Merk range proof @@ -1349,7 +1495,8 @@ impl GroveDb { cost, crate::query::axis_lowering::axis_bounded_merk_query(axis_query) ); - let mut sec_result = cost_return_on_error!( + let secondary_query_for_rows = secondary_query.clone(); + let sec_result = cost_return_on_error!( &mut cost, secondary_merk .prove_without_encoding(secondary_query, Some(*limit), grove_version) @@ -1357,21 +1504,22 @@ impl GroveDb { "axis descent: secondary range proof: {e}" ))) ); - cost_return_on_error!( + let mut serialized = Vec::with_capacity(128); + encode_into(sec_result.proof.iter(), &mut serialized); + target_chains = cost_return_on_error!( &mut cost, - super::reference_resolution::resolve_axis_reference_nodes( - sec_result.proof.iter_mut(), - axis, - &primary_merk, + build_row_target_chains( self, + axis, + &serialized, + secondary_query_for_rows, + Some(*limit), &path_keys, transaction, batch, grove_version, ) ); - let mut serialized = Vec::with_capacity(128); - encode_into(sec_result.proof.iter(), &mut serialized); serialized } } @@ -1401,6 +1549,7 @@ impl GroveDb { primary_root_hash, rank, secondary_proof, + target_chains, }) .wrap_with_cost(cost) } @@ -1412,7 +1561,6 @@ impl GroveDb { #[allow(clippy::too_many_arguments)] fn build_paginated_secondary_proof<'db, S>( secondary_merk: &grovedb_merk::Merk, - primary_merk: &grovedb_merk::Merk, grovedb: &'db GroveDb, primary_path: &[Vec], transaction: &'db Transaction, @@ -1423,7 +1571,7 @@ fn build_paginated_secondary_proof<'db, S>( axis: IndexAxis, grove_version: &GroveVersion, cost: &mut OperationCost, -) -> Result, Error> +) -> Result<(Vec, Vec), Error> where S: grovedb_storage::StorageContext<'db>, { @@ -1435,7 +1583,7 @@ where ))); } let inner_range = MerkQueryItemForRange::RangeFull(std::ops::RangeFull); - let mut prove_result = secondary_merk + let prove_result = secondary_merk .prove_count_offset_on_range( &inner_range, offset, @@ -1447,22 +1595,22 @@ where .map_err(|e| { Error::CorruptedData(format!("axis descent: secondary count-offset proof: {e}")) })?; - // Resolve canonical rows into the primary values they point at, - // before encoding. - super::reference_resolution::resolve_axis_reference_nodes( - prove_result.ops.iter_mut(), - axis, - primary_merk, + let mut serialized = Vec::with_capacity(128); + encode_into(prove_result.ops.iter(), &mut serialized); + let chains = build_paginated_target_chains( grovedb, + axis, + &serialized, + offset, + k, + descending, primary_path, transaction, batch, grove_version, ) .unwrap_add_cost(cost)?; - let mut serialized = Vec::with_capacity(128); - encode_into(prove_result.ops.iter(), &mut serialized); - Ok(serialized) + Ok((serialized, chains)) } /// The secondary-side aggregate proof for diff --git a/grovedb/src/operations/proof/indexed_axis/mod.rs b/grovedb/src/operations/proof/indexed_axis/mod.rs index 15f846b50..f5c40113c 100644 --- a/grovedb/src/operations/proof/indexed_axis/mod.rs +++ b/grovedb/src/operations/proof/indexed_axis/mod.rs @@ -65,13 +65,13 @@ mod axis_api; mod envelope; #[cfg(feature = "minimal")] mod generate; -pub(crate) mod reference_resolution; +pub(crate) mod target_chain; pub(crate) mod verify; pub use envelope::{ AncestorAttestation, AxisEntries, IndexedAxisAggregateProof, IndexedAxisAggregateResult, IndexedAxisPaginatedProof, IndexedAxisPaginatedResult, IndexedAxisQueryResult, - IndexedAxisRangeProof, + IndexedAxisRangeProof, IndexedTargetChain, IndexedTargetCommitment, IndexedTargetNode, }; use grovedb_element::indexed::IndexAxis; diff --git a/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs b/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs deleted file mode 100644 index c30f6c7b7..000000000 --- a/grovedb/src/operations/proof/indexed_axis/reference_resolution.rs +++ /dev/null @@ -1,360 +0,0 @@ -//! Resolving canonical secondary rows into proof nodes that carry the -//! primary value they point at. -//! -//! Every indexed secondary row is a canonical one-hop -//! `ReferenceWithSumItem(SiblingReference(primary_key), Some(1), sum)` -//! (see [`axis_row_reference`]). Merk emits such a row as a -//! `KVValueHashFeatureType` node carrying the REFERENCE's bytes, and -//! leaves dereferencing to the layer above — the same division of labour -//! the regular count-tree proof flow uses. This module is that layer for -//! indexed axes. -//! -//! # The logical-origin rule -//! -//! A secondary Merk lives at a derived storage prefix -//! (`blake3(primary_prefix ‖ axis_tag)`) and has **no `SubtreePath`** — -//! there is no GroveDB path that names it, so the generic path-keyed -//! reference machinery (`follow_reference` / `MerkCache`) cannot express -//! a row's origin at all. Resolution here is therefore purpose-built: -//! a row's logical origin is the INDEXED PRIMARY's path, so -//! `SiblingReference(primary_key)` resolves to `primary_key` in the -//! primary Merk. Nothing in this module treats the derived secondary -//! prefix as a user-visible path, and no caller may. -//! -//! # What the rewrite binds -//! -//! The emitted node is `KVRefValueHash{Count,CountSum}(key, -//! target_bytes, H(reference bytes), ..)`, whose hash reconstruction is -//! `combine_hash(H(reference bytes), H(target_bytes))`. That is exactly -//! the row's committed value hash, so the resolved target value is bound -//! into the secondary root — which the axis verifier in turn binds to the -//! indexed element via `combine_hash_three`. Substituting either the -//! target value or the reference breaks the root. -//! -//! # Immediate-node binding -//! -//! Resolution stops at the primary entry. It does NOT follow a chain to a -//! terminal, because a canonical row binds the IMMEDIATE primary node's -//! committed value hash — see [`INDEXED_SECONDARY_MAX_HOP`]. Ordinary -//! user references keep their terminal semantics; this one-hop rule is -//! selected explicitly here and applies only to indexed rows. -//! -//! [`axis_row_reference`]: crate::operations::indexed_tree::axis_row_reference -//! [`INDEXED_SECONDARY_MAX_HOP`]: -//! crate::operations::indexed_tree::INDEXED_SECONDARY_MAX_HOP - -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; -use grovedb_element::indexed::IndexAxis; -use grovedb_merk::{ - element::{costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions}, - proofs::{Node, Op}, - tree::{combine_hash, value_hash}, - Merk, TreeFeatureType, -}; -use grovedb_storage::{StorageBatch, StorageContext}; -use grovedb_version::version::GroveVersion; - -use crate::{ - operations::indexed_tree::{axis_sort_key_len, indexed_row_target_key}, - Element, Error, GroveDb, Transaction, -}; - -/// Rewrite every canonical reference row in a secondary proof's op stream -/// into a resolved-value node. -/// -/// `primary_merk` must be the indexed primary the secondary mirrors — -/// that is the row's logical origin, and reading the target anywhere else -/// would resolve a `SiblingReference` against the wrong parent. -/// -/// Rows that are not references are left untouched, so this is safe to -/// run over any secondary proof. -#[allow(clippy::too_many_arguments)] -pub(crate) fn resolve_axis_reference_nodes<'a, 'db, S: StorageContext<'db>>( - ops: impl IntoIterator, - axis: IndexAxis, - primary_merk: &Merk, - grovedb: &'db GroveDb, - primary_path: &[Vec], - transaction: &'db Transaction, - batch: &'db StorageBatch, - grove_version: &GroveVersion, -) -> CostResult<(), Error> { - let mut cost = OperationCost::default(); - let sort_len = axis_sort_key_len(axis); - - for op in ops { - let node = match op { - Op::Push(node) | Op::PushInverted(node) => node, - _ => continue, - }; - let Node::KVValueHashFeatureType(key, value, _committed_value_hash, feature_type) = node - else { - continue; - }; - - // Only canonical rows are rewritten. Anything else in a secondary - // is corruption, but this pass is not the checker for that — the - // node is left alone and the shape checks reject it downstream. - let Ok(row) = Element::deserialize(value.as_slice(), grove_version) else { - continue; - }; - if !row.is_reference() { - continue; - } - if key.len() < sort_len { - return Err(Error::CorruptedData(format!( - "indexed-axis reference resolution: secondary key {} is shorter than the \ - {axis:?} axis sort key ({sort_len} bytes)", - hex::encode(key) - ))) - .wrap_with_cost(cost); - } - let key_suffix = key[sort_len..].to_vec(); - // Enforces the canonical shape AND that the reference target - // agrees with the secondary-key suffix. Both are needed: the - // suffix is what the verifier decodes the primary key from, so a - // row whose commitment points elsewhere would authenticate a - // value filed under a key it does not belong to. - let (target_key, _carried_sum) = cost_return_on_error!( - &mut cost, - indexed_row_target_key(&row, &key_suffix, "indexed-axis reference resolution") - .wrap_with_cost(OperationCost::default()) - ); - let target_key = target_key.to_vec(); - - let (target_element, target_value_hash) = cost_return_on_error!( - &mut cost, - Element::get_with_value_hash(primary_merk, &target_key, true, grove_version).map_err( - |e| Error::CorruptedData(format!( - "indexed-axis reference resolution: reading primary entry {}: {e}", - hex::encode(&target_key) - )) - ) - ); - - let target_bytes = cost_return_on_error!( - &mut cost, - target_element - .serialize(grove_version) - .map_err(|e| Error::CorruptedData(format!( - "indexed-axis reference resolution: serializing primary entry {}: {e}", - hex::encode(&target_key) - ))) - .wrap_with_cost(OperationCost::default()) - ); - - // The reference element's OWN value hash — not the node's stored - // (combined) hash. This is the half of the commitment the node - // carries explicitly; the other half the verifier recomputes. - let reference_element_hash = value_hash(value.as_slice()).unwrap_add_cost(&mut cost); - - // Which node family can express this target's commitment? - // - // `KVRefValueHashCountSum` reconstructs the row's hash as - // `combine_hash(H(reference), H(target_bytes))`, which is correct - // only when the target's own committed hash IS `H(target_bytes)` — - // true for item-shaped entries. A tree-shaped entry (the NORMAL - // case under a count-indexed primary, whose whole purpose is - // indexing its children's counts) commits a combined hash instead, - // and no choice of target bytes makes `H(bytes)` equal it. Those - // get the target-witness variant, which carries the missing - // child commitment so the verifier can rebuild both combines. - let simple_target_hash = value_hash(&target_bytes).unwrap_add_cost(&mut cost); - let target_child_hash = if simple_target_hash == target_value_hash { - None - } else { - // Recover the child commitment the target folded in. It is - // read from the target's own subtree root rather than derived - // from the committed hash (which is one-way), and then checked - // against that committed hash below — so a wrong witness is - // caught here rather than becoming an unverifiable proof. - let child_hash = cost_return_on_error!( - &mut cost, - target_layered_child_hash( - &target_element, - primary_path, - &target_key, - grovedb, - transaction, - batch, - grove_version, - ) - ); - let rebuilt = combine_hash(&simple_target_hash, &child_hash).unwrap_add_cost(&mut cost); - if rebuilt != target_value_hash { - return Err(Error::NotSupported(format!( - "indexed-axis proofs cannot yet witness the committed value hash of primary \ - entry {} ({}): its stored hash is a combined hash that \ - combine_hash(H(value), child_root) does not reproduce, so no single \ - child-hash witness expresses it. Reads, writes and integrity verification \ - handle this entry normally; only proving it is unsupported.", - hex::encode(&target_key), - target_element.type_str() - ))) - .wrap_with_cost(cost); - } - Some(child_hash) - }; - - *node = match (feature_type, target_child_hash) { - ( - TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(count, sum), - Some(child_hash), - ) => Node::KVRefValueHashCountSumWithTargetChildHash( - key.clone(), - target_bytes, - reference_element_hash, - *count, - *sum, - child_hash, - ), - (TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(count, sum), None) => { - Node::KVRefValueHashCountSum( - key.clone(), - target_bytes, - reference_element_hash, - *count, - *sum, - ) - } - (TreeFeatureType::ProvableCountedMerkNode(count), None) => { - Node::KVRefValueHashCount(key.clone(), target_bytes, reference_element_hash, *count) - } - (TreeFeatureType::ProvableCountedMerkNode(_), Some(_)) => { - return Err(Error::NotSupported( - "indexed-axis proofs do not support a layered target under a count-only \ - secondary; every axis secondary is a dual-aggregate \ - ProvableCountProvableSumTree, so this shape should be unreachable" - .to_string(), - )) - .wrap_with_cost(cost); - } - (other, _) => { - return Err(Error::CorruptedData(format!( - "indexed-axis reference resolution: secondary row {} carries feature type \ - {other:?}; every axis secondary is a dual-aggregate \ - ProvableCountProvableSumTree, so its rows must be count-bearing", - hex::encode(key) - ))) - .wrap_with_cost(cost); - } - }; - } - - Ok(()).wrap_with_cost(cost) -} - -/// The child commitment a LAYERED primary entry folded into its stored -/// value hash — the second half of -/// `combine_hash(H(element bytes), child)`. -/// -/// Two shapes reach here: -/// -/// - **Subtree-ish elements** (`Tree`, `SumTree`, the `Provable*` family, -/// and the non-Merk trees): the child is the subtree's root hash, read -/// by opening the subtree. -/// - **Reference-shaped entries**: the child is the NEXT target's -/// committed value hash, read one hop along. -/// -/// An indexed-tree element commits with `combine_hash_three` (value, -/// primary root, axes digest) and therefore has no single child witness; -/// it is rejected by the caller's rebuild check rather than here, so the -/// rejection is driven by the hash not reconstructing rather than by a -/// type list that could drift. -#[allow(clippy::too_many_arguments)] -fn target_layered_child_hash<'db>( - target_element: &Element, - primary_path: &[Vec], - target_key: &[u8], - grovedb: &'db GroveDb, - transaction: &'db Transaction, - batch: &'db StorageBatch, - grove_version: &GroveVersion, -) -> CostResult { - let mut cost = OperationCost::default(); - - if let Element::Reference(reference_path, ..) - | Element::ReferenceWithSumItem(reference_path, ..) = target_element - { - // One hop only: the primary entry's own commitment folds in its - // IMMEDIATE target's hash, so that is the witness — following - // further would bind something the primary node does not. - let mut qualified = primary_path.to_vec(); - qualified.push(target_key.to_vec()); - let next_absolute = match crate::reference_path::path_from_reference_path_type( - reference_path.clone(), - &qualified, - Some(target_key), - ) { - Ok(p) => p, - Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), - }; - let Some((next_key, next_parent)) = next_absolute.split_last() else { - return Err(Error::CorruptedData( - "indexed-axis reference resolution: a reference-shaped primary entry resolves \ - to an empty path" - .to_string(), - )) - .wrap_with_cost(cost); - }; - let next_parent_refs: Vec<&[u8]> = next_parent.iter().map(|s| s.as_slice()).collect(); - let next_merk = cost_return_on_error!( - &mut cost, - grovedb.open_transactional_merk_at_path( - next_parent_refs.as_slice().into(), - transaction, - Some(batch), - grove_version, - ) - ); - let next_value_hash = cost_return_on_error!( - &mut cost, - next_merk - .get_value_hash( - next_key.as_slice(), - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|e| Error::CorruptedData(format!( - "indexed-axis reference resolution: reading the next hop's value hash: {e}" - ))) - ); - let resolved = match next_value_hash { - Some(h) => h, - None => { - return Err(Error::CorruptedReferencePathKeyNotFound(format!( - "indexed-axis reference resolution: a reference-shaped primary entry points \ - at {}, which does not exist", - hex::encode(next_key) - ))) - .wrap_with_cost(cost); - } - }; - return Ok(resolved).wrap_with_cost(cost); - } - - // Subtree-ish: the child is the subtree's own root hash. - let mut subtree_path = primary_path.to_vec(); - subtree_path.push(target_key.to_vec()); - let subtree_refs: Vec<&[u8]> = subtree_path.iter().map(|s| s.as_slice()).collect(); - let subtree = cost_return_on_error!( - &mut cost, - grovedb.open_transactional_merk_at_path( - subtree_refs.as_slice().into(), - transaction, - Some(batch), - grove_version, - ) - ); - let (root_hash, ..) = cost_return_on_error!( - &mut cost, - subtree - .root_hash_key_and_aggregate_data() - .map_err(|e| Error::CorruptedData(format!( - "indexed-axis reference resolution: reading a layered primary entry's subtree \ - root: {e}" - ))) - ); - Ok(root_hash).wrap_with_cost(cost) -} diff --git a/grovedb/src/operations/proof/indexed_axis/target_chain.rs b/grovedb/src/operations/proof/indexed_axis/target_chain.rs new file mode 100644 index 000000000..ddc74ee2c --- /dev/null +++ b/grovedb/src/operations/proof/indexed_axis/target_chain.rs @@ -0,0 +1,354 @@ +//! Building and authenticating a secondary row's resolved-target chain. +//! +//! A canonical secondary row is a one-hop reference to its primary entry, +//! committed as `combine_hash(H(row bytes), primary_node_commitment)`. +//! A chain hands the verifier the pieces it needs to rebuild that +//! commitment and, when the primary entry is itself a reference, to keep +//! rebuilding through to the terminal value. +//! +//! # Why this carries no path proofs +//! +//! Each chain entry's commitment is reconstructed from its own bytes plus +//! the NEXT entry's commitment, and the head's commitment is what the row +//! binds. The row's hash is bound into the secondary root, which the axis +//! verifier binds to the indexed element, which chains to the grove root +//! — so substituting any value anywhere in the chain moves the root and +//! is rejected. +//! +//! This is the same trust model shipped GroveDB reference proofs already +//! use: `KVRefValueHash*` binds a reference's committed target hash to the +//! returned value without separately proving the target's path inclusion. +//! A chain is therefore neither weaker nor stronger than reading the same +//! reference through an ordinary proof, and it costs one value plus a +//! hash per row instead of a full inclusion proof per row. +//! +//! A row whose committed target hash disagrees with what actually lives +//! at the referenced path is stale state, not an unsound proof — and +//! `verify_grovedb`'s stale-target-hash check is what detects that. + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; +use grovedb_element::indexed::IndexAxis; +use grovedb_merk::{ + element::get::ElementFetchFromStorageExtensions, + tree::{axes_digest, combine_hash, combine_hash_three, value_hash}, + CryptoHash, +}; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; +use grovedb_storage::StorageBatch; +use grovedb_version::version::GroveVersion; + +use super::{IndexedTargetChain, IndexedTargetCommitment, IndexedTargetNode}; +use crate::{ + operations::MAX_REFERENCE_HOPS, reference_path::path_from_reference_path_type, Element, Error, + GroveDb, Transaction, +}; + +/// Derive the commitment shape for one node, reading whatever roots that +/// shape folds in. +fn build_commitment<'db>( + grovedb: &'db GroveDb, + element: &Element, + qualified_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult { + let mut cost = OperationCost::default(); + let underlying = element.underlying(); + + if underlying.is_reference() { + return Ok(IndexedTargetCommitment::Reference).wrap_with_cost(cost); + } + if !underlying.is_any_tree() { + return Ok(IndexedTargetCommitment::Simple).wrap_with_cost(cost); + } + + let path_owned: SubtreePathBuilder> = + SubtreePathBuilder::owned_from_iter(qualified_path.iter().cloned()); + let node_path = SubtreePath::from(&path_owned); + + // Read the primary/inner root every tree-ish shape folds in. + let inner_root = |cost: &mut OperationCost| -> Result { + let merk = grovedb + .open_transactional_merk_at_path( + node_path.clone(), + transaction, + Some(batch), + grove_version, + ) + .unwrap_add_cost(cost)?; + merk.root_hash_key_and_aggregate_data() + .unwrap_add_cost(cost) + .map(|(hash, ..)| hash) + .map_err(Error::MerkError) + }; + + let commitment = match underlying { + Element::ProvableCountIndexedTree(_, secondary_root_key, ..) + | Element::ProvableSumIndexedTree(_, secondary_root_key, ..) => { + let axis = match underlying { + Element::ProvableCountIndexedTree(..) => IndexAxis::Count, + _ => IndexAxis::Sum, + }; + let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); + let secondary = cost_return_on_error!( + &mut cost, + grovedb.open_indexed_secondary_at_path( + node_path.clone(), + axis, + secondary_root_key.clone(), + transaction, + Some(batch), + grove_version, + ) + ); + let (secondary_root_hash, ..) = cost_return_on_error!( + &mut cost, + secondary + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) + ); + IndexedTargetCommitment::IndexedSingle { + primary_root_hash, + secondary_root_hash, + } + } + Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => { + let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); + let mut axis_hashes: Vec<(u8, CryptoHash)> = Vec::with_capacity(axes.len()); + for (tag, secondary_root_key) in axes { + let axis = cost_return_on_error_no_add!( + cost, + IndexAxis::try_from_tag(*tag).map_err(|e| Error::CorruptedData(format!( + "indexed target chain: invalid PCPSIT axis tag: {e}" + ))) + ); + let secondary = cost_return_on_error!( + &mut cost, + grovedb.open_indexed_secondary_at_path( + node_path.clone(), + axis, + secondary_root_key.clone(), + transaction, + Some(batch), + grove_version, + ) + ); + let (secondary_root_hash, ..) = cost_return_on_error!( + &mut cost, + secondary + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) + ); + axis_hashes.push((*tag, secondary_root_hash)); + } + IndexedTargetCommitment::IndexedMulti { + primary_root_hash, + axes: axis_hashes, + } + } + // Every other tree-ish shape — ordinary/sum/count Merk trees and + // the non-Merk append trees — commits as + // `combine_hash(H(value), child_root)`. Non-Merk trees keep their + // state root in the same position, so they need no special case. + _ => { + let child_root = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); + IndexedTargetCommitment::Layered(child_root) + } + }; + Ok(commitment).wrap_with_cost(cost) +} + +/// Build the chain for one secondary row: the immediate primary entry, +/// then any ordinary reference hops through to the terminal. +pub(crate) fn build_target_chain<'db>( + grovedb: &'db GroveDb, + indexed_path: &[Vec], + primary_key: &[u8], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult { + let mut cost = OperationCost::default(); + let mut qualified_path = indexed_path.to_vec(); + qualified_path.push(primary_key.to_vec()); + let mut visited = std::collections::HashSet::new(); + let mut nodes = Vec::new(); + + for _ in 0..=MAX_REFERENCE_HOPS { + if !visited.insert(qualified_path.clone()) { + return Err(Error::CyclicReference).wrap_with_cost(cost); + } + let Some((key, parent_segments)) = qualified_path.split_last() else { + return Err(Error::CorruptedPath( + "indexed target chain resolved an empty path".to_string(), + )) + .wrap_with_cost(cost); + }; + let parent_slices: Vec<&[u8]> = parent_segments.iter().map(Vec::as_slice).collect(); + let parent_merk = cost_return_on_error!( + &mut cost, + grovedb.open_transactional_merk_at_path( + parent_slices.as_slice().into(), + transaction, + Some(batch), + grove_version, + ) + ); + let element = cost_return_on_error!( + &mut cost, + Element::get(&parent_merk, key, true, grove_version).map_err(|e| { + Error::CorruptedData(format!( + "indexed target chain: entry {} is missing: {e}", + hex::encode(key) + )) + }) + ); + drop(parent_merk); + + let value = cost_return_on_error_no_add!( + cost, + element.serialize(grove_version).map_err(|e| { + Error::CorruptedData(format!("indexed target chain: serializing entry: {e}")) + }) + ); + let commitment = cost_return_on_error!( + &mut cost, + build_commitment( + grovedb, + &element, + &qualified_path, + transaction, + batch, + grove_version, + ) + ); + + let next_path = match element.underlying() { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => { + let mut current = parent_segments.to_vec(); + current.push(key.clone()); + match path_from_reference_path_type( + reference_path.clone(), + ¤t, + Some(key.as_slice()), + ) { + Ok(p) => Some(p), + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + } + } + _ => None, + }; + + nodes.push(IndexedTargetNode { value, commitment }); + match next_path { + Some(next) => qualified_path = next, + None => return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost), + } + } + + Err(Error::ReferenceLimit).wrap_with_cost(cost) +} + +/// Recompute the committed value hash of one chain entry, given the +/// commitment of the entry that follows it. +fn node_commitment( + node: &IndexedTargetNode, + next: Option<&CryptoHash>, +) -> Result { + let serialized_hash = value_hash(&node.value).value().to_owned(); + Ok(match &node.commitment { + IndexedTargetCommitment::Simple => serialized_hash, + IndexedTargetCommitment::Layered(child_root) => combine_hash(&serialized_hash, child_root) + .value() + .to_owned(), + IndexedTargetCommitment::IndexedSingle { + primary_root_hash, + secondary_root_hash, + } => combine_hash_three(&serialized_hash, primary_root_hash, secondary_root_hash) + .value() + .to_owned(), + IndexedTargetCommitment::IndexedMulti { + primary_root_hash, + axes, + } => { + let digest = axes_digest(axes).value().to_owned(); + combine_hash_three(&serialized_hash, primary_root_hash, &digest) + .value() + .to_owned() + } + IndexedTargetCommitment::Reference => { + let next = next.ok_or_else(|| { + Error::CorruptedData( + "indexed target chain: a Reference node is the last entry — a reference \ + commits its target's hash, so the chain must continue" + .to_string(), + ) + })?; + combine_hash(&serialized_hash, next).value().to_owned() + } + }) +} + +/// Authenticate a chain and return `(immediate primary element, its +/// committed value hash, terminal element)`. +/// +/// The caller checks the returned commitment against what the row bound; +/// this function only proves the chain is internally consistent and +/// decodes its ends. +pub(crate) fn authenticate_target_chain( + chain: &IndexedTargetChain, + grove_version: &GroveVersion, +) -> Result<(Element, CryptoHash, Element), Error> { + if chain.nodes.is_empty() { + return Err(Error::CorruptedData( + "indexed target chain is empty — every row resolves to at least its immediate \ + primary entry" + .to_string(), + )); + } + if chain.nodes.len() > MAX_REFERENCE_HOPS + 1 { + return Err(Error::CorruptedData(format!( + "indexed target chain has {} entries, above the {MAX_REFERENCE_HOPS}-hop limit", + chain.nodes.len() + ))); + } + // Only the last entry may be a non-reference, and only a reference may + // be followed — otherwise a prover could append unbound entries after + // a terminal and pass off the last one as the resolved value. + for (i, node) in chain.nodes.iter().enumerate() { + let is_last = i + 1 == chain.nodes.len(); + let is_reference = matches!(node.commitment, IndexedTargetCommitment::Reference); + if is_reference == is_last { + return Err(Error::CorruptedData(format!( + "indexed target chain entry {i} is {} but {} the last — a chain is a run of \ + references ending in exactly one non-reference terminal", + if is_reference { + "a reference" + } else { + "not a reference" + }, + if is_last { "is" } else { "is not" } + ))); + } + } + + // Fold back-to-front: each entry's commitment needs the next one's. + let mut next: Option = None; + for node in chain.nodes.iter().rev() { + next = Some(node_commitment(node, next.as_ref())?); + } + let immediate_commitment = next.expect("chain is non-empty"); + + let decode = |bytes: &[u8]| -> Result { + Element::deserialize(bytes, grove_version).map_err(|e| { + Error::CorruptedData(format!("indexed target chain: undecodable element: {e}")) + }) + }; + let immediate = decode(&chain.nodes[0].value)?; + let terminal = decode(&chain.nodes[chain.nodes.len() - 1].value)?; + Ok((immediate, immediate_commitment, terminal)) +} diff --git a/grovedb/src/operations/proof/indexed_axis/verify.rs b/grovedb/src/operations/proof/indexed_axis/verify.rs index c2f55befe..f85e06066 100644 --- a/grovedb/src/operations/proof/indexed_axis/verify.rs +++ b/grovedb/src/operations/proof/indexed_axis/verify.rs @@ -21,12 +21,12 @@ use grovedb_merk::{ use grovedb_query::{AggregateFold, QueryItem as MerkQueryItem}; use grovedb_version::{check_grovedb_v0, version::GroveVersion}; -use crate::{Element, Error, GroveDb}; +use crate::{query_result_type::IndexedAxisEntry, Element, Error, GroveDb}; use super::{ aggregate_range_out_of_domain, AncestorAttestation, AxisEntries, IndexedAxisAggregateProof, IndexedAxisAggregateResult, IndexedAxisPaginatedProof, IndexedAxisPaginatedResult, - IndexedAxisQueryResult, IndexedAxisRangeProof, + IndexedAxisQueryResult, IndexedAxisRangeProof, IndexedTargetChain, }; /// Walk the verifier-side ancestor chain (depths `last_idx - 1` down to @@ -494,9 +494,9 @@ impl GroveDb { ))); } let yielded_key: &[u8] = match &result.entries { - AxisEntries::Count(v) if v.len() == 1 => &v[0].1, - AxisEntries::Sum(v) if v.len() == 1 => &v[0].1, - AxisEntries::Avg(v) if v.len() == 1 => &v[0].1, + AxisEntries::Count(v) if v.len() == 1 => &v[0].primary_key, + AxisEntries::Sum(v) if v.len() == 1 => &v[0].primary_key, + AxisEntries::Avg(v) if v.len() == 1 => &v[0].primary_key, other => { return Err(Error::CorruptedData(format!( "indexed-axis rank proof: expected exactly one yielded entry at the rank \ @@ -666,7 +666,12 @@ fn verify_indexed_axis_range_inner( "indexed-axis range proof", )?; - let entries = decode_axis_entries_from_result_set(axis, &sec_result.result_set, grove_version)?; + let entries = decode_axis_entries_from_result_set( + axis, + &sec_result.result_set, + &envelope.target_chains, + grove_version, + )?; Ok(IndexedAxisQueryResult { root_hash, entries }) } @@ -713,6 +718,7 @@ fn verify_indexed_axis_paginated_inner( let entries = decode_axis_entries_from_count_offset_items( axis, &count_offset_result.returned_items, + &envelope.target_chains, grove_version, )?; let (secondary_root_hash, skipped) = @@ -829,36 +835,37 @@ fn verify_indexed_axis_aggregate_inner( }) } -/// Authenticate ONE resolved secondary row and decode it. -/// -/// Everything needed is already in the proof, and all of it is bound to -/// the secondary root: the row's key, its committed reference hash, and -/// the resolved target bytes. From the AUTHENTICATED target element we -/// re-derive the `(count, sum)` the mirror would have seen, rebuild both -/// the secondary key and the canonical row those aggregates imply, and -/// check them against what the proof carried. +/// Authenticate ONE secondary row against its resolved-target chain and +/// decode it into an entry. /// -/// That single comparison authenticates every part of the row at once: +/// Two independent things are checked, and both are needed: /// -/// - the ordering prefix (it is part of the rebuilt secondary key), -/// - the primary key in the suffix (likewise), -/// - the reference path — that it is `SiblingReference(primary_key)` and -/// not a reference to some other entry, -/// - the one-hop budget, -/// - the carried payload sum. +/// 1. **The row is canonical.** From the chain's IMMEDIATE primary +/// element we re-derive the `(count, sum)` the mirror would have seen, +/// rebuild the secondary key and the canonical row those aggregates +/// imply, and compare against what the proof carried. That covers the +/// ordering prefix, the primary-key suffix, the reference path, the +/// one-hop budget and the carried payload sum in one comparison — so a +/// row filed under `…‖a` whose reference points at `b` is rejected +/// rather than assumed away. +/// 2. **The chain is what the row committed.** The row's recorded value +/// hash must equal `combine_hash(H(row bytes), immediate commitment)`, +/// where the immediate commitment is folded back from the chain. Since +/// the row's hash is bound into the secondary root, this is what makes +/// the returned value unforgeable. /// -/// Without it a verifier would be trusting that a committed reference -/// points at the key encoded in the row it sits in. It does not have to: -/// a row filed under `…‖a` whose reference points at `b` produces a -/// different canonical-row hash, and is rejected here. +/// Returns the terminal value: a reference-shaped primary entry resolves +/// through to what `db.get` on that key would give, while remaining BOUND +/// through its immediate node. fn authenticate_axis_row( axis: IndexAxis, secondary_key: &[u8], - target_bytes: &[u8], - reference_element_hash: grovedb_merk::CryptoHash, + row_bytes: &[u8], + row_recorded_value_hash: CryptoHash, + chain: &IndexedTargetChain, grove_version: &GroveVersion, ) -> Result<(Vec, Element), Error> { - use grovedb_merk::tree::value_hash; + use grovedb_merk::tree::{combine_hash, value_hash}; use crate::operations::indexed_tree::{ axis_payload_sum, axis_row_reference, axis_sort_key_len, make_axis_secondary_key, @@ -873,19 +880,15 @@ fn authenticate_axis_row( } let primary_key = secondary_key[sort_len..].to_vec(); - let element = Element::deserialize(target_bytes, grove_version).map_err(|e| { - Error::CorruptedData(format!( - "indexed-axis proof returned a non-Element primary value at key {}: {e}", - hex::encode(&primary_key) - )) - })?; - let (count, sum) = element.count_sum_value_or_default(); + let (immediate, immediate_commitment, terminal) = + super::target_chain::authenticate_target_chain(chain, grove_version)?; + let (count, sum) = immediate.count_sum_value_or_default(); let expected_secondary_key = make_axis_secondary_key(axis, count, sum, &primary_key); if expected_secondary_key != secondary_key { return Err(Error::CorruptedData(format!( - "indexed-axis ({axis:?}) row is filed under {} but the primary value it resolves \ - to implies {} — the row's sort position does not match its own value", + "indexed-axis ({axis:?}) row is filed under {} but the primary value it resolves to \ + implies {} — the row's sort position does not match its own value", hex::encode(secondary_key), hex::encode(&expected_secondary_key) ))); @@ -895,70 +898,90 @@ fn authenticate_axis_row( let expected_row_bytes = expected_row.serialize(grove_version).map_err(|e| { Error::CorruptedData(format!("serializing the expected canonical axis row: {e}")) })?; - let expected_reference_hash = value_hash(&expected_row_bytes).unwrap(); - if expected_reference_hash != reference_element_hash { + if row_bytes != expected_row_bytes.as_slice() { return Err(Error::CorruptedData(format!( - "indexed-axis ({axis:?}) row at {} does not commit the canonical reference for its \ - primary key — expected a one-hop SiblingReference({}) carrying sum {}", + "indexed-axis ({axis:?}) row at {} is not the canonical one-hop reference for its \ + primary key — expected SiblingReference({}) carrying sum {}", hex::encode(secondary_key), hex::encode(&primary_key), axis_payload_sum(axis, count, sum)? ))); } - Ok((primary_key, element)) + let expected_committed = combine_hash(value_hash(row_bytes).value(), &immediate_commitment) + .value() + .to_owned(); + if expected_committed != row_recorded_value_hash { + return Err(Error::CorruptedData(format!( + "indexed-axis ({axis:?}) row at {} is bound to a different primary commitment than \ + its target chain reconstructs — the row is stale or the chain was substituted", + hex::encode(secondary_key) + ))); + } + + Ok((primary_key, terminal)) } -/// Build the per-axis entry list from authenticated `(secondary_key, -/// target_bytes, reference_element_hash)` triples. -/// -/// The resolved primary Element is authenticated here and then dropped: -/// the public result type still carries `(ordering_value, primary_key)`. -/// Surfacing the resolved value through the read and result APIs is a -/// separate, purely mechanical change (issue #814 Phase 3) and is -/// deliberately not bundled with the representation change — but the -/// authentication below is NOT deferrable, because without it a verifier -/// would be assuming that a committed reference points at the key encoded -/// in the row it sits in. -fn axis_entries_from_authenticated_rows( +/// Build the per-axis entry list from `(secondary_key, row_bytes, +/// recorded_value_hash)` triples plus their positionally-matched chains. +fn axis_entries_from_rows( axis: IndexAxis, - rows: impl IntoIterator, Vec, grovedb_merk::CryptoHash)>, + rows: Vec<(Vec, Vec, CryptoHash)>, + chains: &[IndexedTargetChain], grove_version: &GroveVersion, ) -> Result { use grovedb_element::indexed::sort_keys::{ decode_avg_sort_key, decode_count_sort_key, decode_sum_sort_key, }; - let mut count_entries: Vec<(u64, Vec)> = Vec::new(); - let mut sum_entries: Vec<(i64, Vec)> = Vec::new(); - let mut avg_entries: Vec<(i128, Vec)> = Vec::new(); + if rows.len() != chains.len() { + return Err(Error::CorruptedData(format!( + "indexed-axis ({axis:?}) proof returned {} rows but carries {} target chains", + rows.len(), + chains.len() + ))); + } + + let mut count_entries = Vec::new(); + let mut sum_entries = Vec::new(); + let mut avg_entries = Vec::new(); - for (secondary_key, target_bytes, reference_element_hash) in rows { + for ((secondary_key, row_bytes, recorded_value_hash), chain) in rows.into_iter().zip(chains) { let (primary_key, value) = authenticate_axis_row( axis, &secondary_key, - &target_bytes, - reference_element_hash, + &row_bytes, + recorded_value_hash, + chain, grove_version, )?; match axis { IndexAxis::Count => { let mut b = [0u8; 8]; b.copy_from_slice(&secondary_key[..8]); - let _ = value; - count_entries.push((decode_count_sort_key(&b), primary_key)); + count_entries.push(IndexedAxisEntry { + ordering_value: decode_count_sort_key(&b), + primary_key, + value, + }); } IndexAxis::Sum => { let mut b = [0u8; 8]; b.copy_from_slice(&secondary_key[..8]); - let _ = value; - sum_entries.push((decode_sum_sort_key(&b), primary_key)); + sum_entries.push(IndexedAxisEntry { + ordering_value: decode_sum_sort_key(&b), + primary_key, + value, + }); } IndexAxis::Avg => { let mut b = [0u8; 16]; b.copy_from_slice(&secondary_key[..16]); - let _ = value; - avg_entries.push((decode_avg_sort_key(&b), primary_key)); + avg_entries.push(IndexedAxisEntry { + ordering_value: decode_avg_sort_key(&b), + primary_key, + value, + }); } } } @@ -972,16 +995,10 @@ fn axis_entries_from_authenticated_rows( /// Decode + authenticate the rows of an ordinary (non-count-offset) axis /// range proof. -/// -/// `proved.proof` is the value hash the merk verifier surfaced for the -/// node. For the resolved-reference nodes an axis proof emits, that is -/// the REFERENCE element's own hash — which is exactly what -/// [`authenticate_axis_row`] needs. A row that was not a resolved -/// reference fails that check, which is the correct outcome: every -/// indexed secondary row must be one. pub(crate) fn decode_axis_entries_from_result_set( axis: IndexAxis, result_set: &[grovedb_merk::proofs::query::ProvedKeyOptionalValue], + chains: &[IndexedTargetChain], grove_version: &GroveVersion, ) -> Result { let rows = result_set @@ -989,43 +1006,28 @@ pub(crate) fn decode_axis_entries_from_result_set( .map(|proved| { let value = proved.value.clone().ok_or_else(|| { Error::CorruptedData(format!( - "indexed-axis ({axis:?}) proof returned no value for secondary key {} — \ - every row resolves to its primary entry", + "indexed-axis ({axis:?}) proof returned no row bytes for secondary key {}", hex::encode(&proved.key) )) })?; Ok((proved.key.clone(), value, proved.proof)) }) .collect::, Error>>()?; - axis_entries_from_authenticated_rows(axis, rows, grove_version) + axis_entries_from_rows(axis, rows, chains, grove_version) } /// Decode + authenticate the rows of a count-offset paginated axis proof. -/// -/// `reference_element_hash` is `Some` exactly when the prover's reference -/// post-pass rewrote the row into a resolved-value node. `None` means the -/// row was surfaced without being resolved, which for an indexed -/// secondary is malformed. pub(crate) fn decode_axis_entries_from_count_offset_items( axis: IndexAxis, items: &[grovedb_merk::proofs::query::CountOffsetReturnedItem], + chains: &[IndexedTargetChain], grove_version: &GroveVersion, ) -> Result { let rows = items .iter() - .map(|it| { - let reference_element_hash = it.reference_element_hash.ok_or_else(|| { - Error::CorruptedData(format!( - "indexed-axis ({axis:?}) paginated proof surfaced an unresolved row at \ - secondary key {} — every row is a canonical reference and must arrive \ - resolved", - hex::encode(&it.key) - )) - })?; - Ok((it.key.clone(), it.value.clone(), reference_element_hash)) - }) - .collect::, Error>>()?; - axis_entries_from_authenticated_rows(axis, rows, grove_version) + .map(|it| (it.key.clone(), it.value.clone(), it.value_hash)) + .collect::>(); + axis_entries_from_rows(axis, rows, chains, grove_version) } pub(crate) fn count_aggregate_inner_range(lo: i128, hi: i128) -> MerkQueryItemForRange { diff --git a/grovedb/src/operations/proof/mod.rs b/grovedb/src/operations/proof/mod.rs index afc6f2b47..01933fa60 100644 --- a/grovedb/src/operations/proof/mod.rs +++ b/grovedb/src/operations/proof/mod.rs @@ -402,6 +402,10 @@ pub struct AxisDescentProof { /// Merk range proof for `Bounded`, an aggregate-on-range proof for /// `AggregateOverValueRange`. pub secondary_proof: Vec, + /// One resolved-target chain per returned secondary row, in the + /// secondary proof's result order. Empty for aggregate traversals, + /// which enumerate no rows. + pub target_chains: Vec, } impl AxisDescentProof { @@ -989,23 +993,6 @@ fn node_to_string(node: &Node) -> Result { hex::encode(value_hash), count ), - Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - count, - sum, - target_child_hash, - ) => format!( - "KVRefValueHashCountSumWithTargetChildHash({}, {}, HASH[{}], {}, {}, \ - target_child=HASH[{}])", - hex_to_ascii(key), - element_hex_to_ascii(value)?, - hex::encode(value_hash), - count, - sum, - hex::encode(target_child_hash) - ), Node::KVRefValueHash(key, value, value_hash) => format!( "KVRefValueHash({}, {}, HASH[{}])", hex_to_ascii(key), diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index eb8932db3..7fceb7024 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -914,6 +914,7 @@ impl GroveDb { let entries = decode_axis_entries_from_count_offset_items( axis, &res.returned_items, + &payload.target_chains, grove_version, )?; ( @@ -963,6 +964,7 @@ impl GroveDb { let entries = decode_axis_entries_from_count_offset_items( axis, &res.returned_items, + &payload.target_chains, grove_version, )?; let yielded_key = entries.first_original_key().map(|k| k.to_vec()); @@ -1016,8 +1018,12 @@ impl GroveDb { format!("axis descent: secondary range proof failed: {e}"), ) })?; - let entries = - decode_axis_entries_from_result_set(axis, &res.result_set, grove_version)?; + let entries = decode_axis_entries_from_result_set( + axis, + &res.result_set, + &payload.target_chains, + grove_version, + )?; ( root, AxisWalkResult::Entries { @@ -4247,10 +4253,7 @@ impl GroveDb { | Node::KVSum(key, value, ..) | Node::KVRefValueHashSum(key, value, ..) | Node::KVCountSum(key, value, ..) - | Node::KVRefValueHashCountSum(key, value, ..) - | Node::KVRefValueHashCountSumWithTargetChildHash(key, value, ..) => { - Some((key.clone(), value.clone())) - } + | Node::KVRefValueHashCountSum(key, value, ..) => Some((key.clone(), value.clone())), // These nodes don't have values, only key+hash or just hash Node::KVDigest(..) | Node::KVDigestCount(..) diff --git a/grovedb/src/query_result_type.rs b/grovedb/src/query_result_type.rs index 6c4deca72..76334071a 100644 --- a/grovedb/src/query_result_type.rs +++ b/grovedb/src/query_result_type.rs @@ -7,6 +7,66 @@ use std::{ }; pub use grovedb_merk::proofs::query::{Key, Path, PathKey}; + +/// One row of a non-aggregate indexed-axis read. +/// +/// Carries the primary VALUE, not just a pointer to it. A secondary row +/// is a canonical reference to its primary entry, so the value comes back +/// with the row — a caller no longer needs `k` follow-up `db.get` calls +/// after a top-k result, nor (for verified reads) `k` extra inclusion +/// proofs. +/// +/// `value` is the entry after ordinary GroveDB reference resolution: if +/// the primary entry is itself a reference, this is its TERMINAL, exactly +/// as reading that key through `db.get` would give you. That is a +/// deliberate asymmetry with how a row is *bound* — a row commits to the +/// immediate primary node, which is what keeps the mirror's invariant +/// local — and the two are consistent because the immediate node's +/// commitment transitively covers the terminal it pointed at when written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedAxisEntry { + /// The value this axis sorts on, decoded from the secondary-key + /// prefix — a count, a sum, or a fixed-point average. + pub ordering_value: T, + /// The primary key, decoded from the secondary-key suffix. + pub primary_key: Vec, + /// The resolved primary value. + pub value: Element, +} + +impl IndexedAxisEntry { + /// The `(ordering_value, primary_key)` pair, dropping the value. + /// + /// For callers that genuinely only rank — leaderboards, ranking views + /// — and for comparing against the pre-resolution shape. + pub fn key_pair(self) -> (T, Vec) { + (self.ordering_value, self.primary_key) + } +} + +/// Compare an entry against a bare `(ordering_value, primary_key)` pair. +/// +/// A migration and ranking-assertion convenience: it deliberately ignores +/// `value`, so it answers "is this row in the right place?" and NOT "does +/// it carry the right value". Tests that care about resolved values must +/// assert on `value` directly — comparing against a tuple will not do it +/// for them. +/// +/// This cannot be confused with ordinary equality: `IndexedAxisEntry`'s +/// own derived `PartialEq` compares every field, and this impl is only +/// reachable when one side is explicitly a tuple. +impl PartialEq<(T, Vec)> for IndexedAxisEntry { + fn eq(&self, other: &(T, Vec)) -> bool { + self.ordering_value == other.0 && self.primary_key == other.1 + } +} + +impl PartialEq> for (T, Vec) { + fn eq(&self, other: &IndexedAxisEntry) -> bool { + self.0 == other.ordering_value && self.1 == other.primary_key + } +} + use grovedb_version::{version::GroveVersion, TryFromVersioned}; use crate::element::SumValue; diff --git a/grovedb/src/tests/axis_descent_proof_tests.rs b/grovedb/src/tests/axis_descent_proof_tests.rs index 06722e0ca..7328365cc 100644 --- a/grovedb/src/tests/axis_descent_proof_tests.rs +++ b/grovedb/src/tests/axis_descent_proof_tests.rs @@ -9,6 +9,8 @@ mod tests { use grovedb_merk::proofs::query::{AggregateFold, AxisQuery, IndexAxis}; use grovedb_version::version::{GroveVersion, GROVE_VERSIONS}; + use crate::IndexedAxisEntry; + use crate::{ operations::proof::{ indexed_axis::AxisEntries, AxisDescentProof, GroveDBProof, LayerProof, ProofBytes, @@ -137,7 +139,7 @@ mod tests { .expect("prove axis path query") } - fn entries_as_sum(entries: &AxisEntries) -> &[(i64, Vec)] { + fn entries_as_sum(entries: &AxisEntries) -> &[IndexedAxisEntry] { match entries { AxisEntries::Sum(entries) => entries, other => panic!("expected sum entries, got {other:?}"), diff --git a/grovedb/src/tests/batch_indexed_fresh_create_tests.rs b/grovedb/src/tests/batch_indexed_fresh_create_tests.rs index 3709fe6ca..28c664fe3 100644 --- a/grovedb/src/tests/batch_indexed_fresh_create_tests.rs +++ b/grovedb/src/tests/batch_indexed_fresh_create_tests.rs @@ -21,6 +21,8 @@ mod tests { use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntry; + use crate::{ batch::QualifiedGroveDbOp, tests::{make_test_grovedb, TempGroveDb, TEST_LEAF}, @@ -458,7 +460,11 @@ mod tests { ); } - fn one_key_count(db: &TempGroveDb, path: &[&[u8]], gv: &GroveVersion) -> Vec<(u64, Vec)> { + fn one_key_count( + db: &TempGroveDb, + path: &[&[u8]], + gv: &GroveVersion, + ) -> Vec> { db.indexed_count_top_k(path, 5, true, None, gv) .unwrap() .expect("count top_k") diff --git a/grovedb/src/tests/batch_indexed_multi_axis_tests.rs b/grovedb/src/tests/batch_indexed_multi_axis_tests.rs index 25a52699d..d0966020d 100644 --- a/grovedb/src/tests/batch_indexed_multi_axis_tests.rs +++ b/grovedb/src/tests/batch_indexed_multi_axis_tests.rs @@ -277,7 +277,7 @@ mod tests { .indexed_avg_top_k(path.as_ref(), 10, true, None, gv) .unwrap() .expect("avg top_k"); - let avg_keys: Vec> = by_avg.iter().map(|(_, k)| k.clone()).collect(); + let avg_keys: Vec> = by_avg.iter().map(|e| e.primary_key.clone()).collect(); assert_eq!( avg_keys, vec![b"a".to_vec(), b"c".to_vec(), b"b".to_vec()], @@ -388,7 +388,7 @@ mod tests { .unwrap() .expect("avg top_k"); assert_eq!(by_avg.len(), 1, "the avg axis must not have a stale row"); - assert_eq!(by_avg[0].1, b"a".to_vec()); + assert_eq!(by_avg[0].primary_key, b"a".to_vec()); assert_clean(&db, gv); } @@ -440,7 +440,7 @@ mod tests { .unwrap() .expect("sum top_k") .into_iter() - .map(|(_, k)| k) + .map(|e| e.primary_key) .collect::>(), ), ( @@ -449,7 +449,7 @@ mod tests { .unwrap() .expect("count top_k") .into_iter() - .map(|(_, k)| k) + .map(|e| e.primary_key) .collect::>(), ), ( @@ -458,7 +458,7 @@ mod tests { .unwrap() .expect("avg top_k") .into_iter() - .map(|(_, k)| k) + .map(|e| e.primary_key) .collect::>(), ), ] { @@ -854,7 +854,7 @@ mod tests { .unwrap() .expect("sum top_k"); assert_eq!(by_sum.len(), n as usize, "every entry must be indexed"); - let sums: Vec = by_sum.iter().map(|(s, _)| *s).collect(); + let sums: Vec = by_sum.iter().map(|e| e.ordering_value).collect(); let mut sorted = sums.clone(); sorted.sort_unstable(); assert_eq!(sums, sorted, "the sum axis must come back ascending"); @@ -1228,10 +1228,33 @@ mod tests { .indexed_avg_top_k([TEST_LEAF, b"idx"].as_ref(), 5, true, None, gv) .unwrap() .expect("avg top_k"); + // The SORT POSITION is unchanged — that is what this test is + // about. The resolved VALUE is not, and asserting both keeps the + // two apart: a reference row that failed to refresh would keep + // reporting the stale `(count 1, sum 5)` child here while its key + // stayed put, which is exactly the drift the sort key cannot see. assert_eq!( - avg_before, avg_after, + avg_before + .iter() + .map(|e| (e.ordering_value, e.primary_key.clone())) + .collect::>(), + avg_after + .iter() + .map(|e| (e.ordering_value, e.primary_key.clone())) + .collect::>(), "the avg sort key is unchanged by (1,5) -> (2,10)" ); + assert_eq!( + avg_before[0].value.count_sum_value_or_default(), + (1, 5), + "before the change the row resolved to the (count 1, sum 5) child" + ); + assert_eq!( + avg_after[0].value.count_sum_value_or_default(), + (2, 10), + "the row must resolve to the UPDATED child even though its avg \ + sort key did not move" + ); assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"idx"].as_ref(), 5, true, None, gv) .unwrap() diff --git a/grovedb/src/tests/coverage_round7_tests.rs b/grovedb/src/tests/coverage_round7_tests.rs index 3cf0a6353..508379682 100644 --- a/grovedb/src/tests/coverage_round7_tests.rs +++ b/grovedb/src/tests/coverage_round7_tests.rs @@ -798,6 +798,7 @@ mod tests { other_axes_root_hashes: vec![], target_is_pcpsit: false, secondary_proof: vec![], + target_chains: Vec::new(), requested_limit: Some(1), descending: true, }; @@ -823,6 +824,7 @@ mod tests { other_axes_root_hashes: vec![], target_is_pcpsit: false, secondary_proof: vec![], + target_chains: Vec::new(), requested_k: 1, requested_offset: 0, descending: true, @@ -1765,7 +1767,7 @@ mod tests { .unwrap() .expect("top-k"); assert_eq!(top.len(), 1); - assert_eq!(top[0].1, b"x".to_vec()); + assert_eq!(top[0].primary_key, b"x".to_vec()); assert_verify_passes(&db, grove_version); } @@ -2206,8 +2208,8 @@ mod tests { }; // Descending starting at offset 1: skip "e"(9), then take "d"(7), "c"(5). assert_eq!(entries.len(), 2); - assert_eq!(entries[0].0, 7); - assert_eq!(entries[1].0, 5); + assert_eq!(entries[0].ordering_value, 7); + assert_eq!(entries[1].ordering_value, 5); } /// PSIT arbitrary query round trip. @@ -2236,8 +2238,8 @@ mod tests { _ => panic!("expected sum"), }; assert_eq!(entries.len(), 2); - assert_eq!(entries[0].0, 10); - assert_eq!(entries[1].0, 5); + assert_eq!(entries[0].ordering_value, 10); + assert_eq!(entries[1].ordering_value, 5); } /// L4019-4028: ops_at_level_above (level above) exists but the diff --git a/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs b/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs index 090fb754f..8a1539bb0 100644 --- a/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs +++ b/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs @@ -19,6 +19,8 @@ mod tests { use grovedb_merk::proofs::query::AggregateFold; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntry; + use crate::{ operations::proof::indexed_axis::{ AncestorAttestation, AxisEntries, IndexedAxisAggregateProof, IndexedAxisRangeProof, @@ -39,7 +41,7 @@ mod tests { assert!(issues.is_empty(), "verify_grovedb issues: {issues:?}"); } - fn count_entries(entries: &AxisEntries) -> &Vec<(u64, Vec)> { + fn count_entries(entries: &AxisEntries) -> &Vec> { match entries { AxisEntries::Count(v) => v, other => panic!("expected count entries, got {other:?}"), @@ -241,8 +243,20 @@ mod tests { ); assert_eq!( result.entries, - AxisEntries::Sum(vec![(9i64, b"y".to_vec()), (-4i64, b"x".to_vec())]), - "descending sum order, negatives sorting below positives" + AxisEntries::Sum(vec![ + IndexedAxisEntry { + ordering_value: 9i64, + primary_key: b"y".to_vec(), + value: Element::new_sum_item(9), + }, + IndexedAxisEntry { + ordering_value: -4i64, + primary_key: b"x".to_vec(), + value: Element::new_sum_item(-4), + }, + ]), + "descending sum order, negatives sorting below positives; each row \ + carries its resolved primary value" ); } diff --git a/grovedb/src/tests/indexed_axis_offset_proof_tests.rs b/grovedb/src/tests/indexed_axis_offset_proof_tests.rs index e81988bdf..5d58eb3f4 100644 --- a/grovedb/src/tests/indexed_axis_offset_proof_tests.rs +++ b/grovedb/src/tests/indexed_axis_offset_proof_tests.rs @@ -27,6 +27,8 @@ mod tests { use grovedb_element::indexed::IndexAxis; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntry; + use crate::{ operations::proof::indexed_axis::AxisEntries, tests::{make_test_grovedb, TEST_LEAF}, @@ -88,7 +90,7 @@ mod tests { } } - fn entries_as_sum(entries: &AxisEntries) -> &[(i64, Vec)] { + fn entries_as_sum(entries: &AxisEntries) -> &[IndexedAxisEntry] { match entries { AxisEntries::Sum(v) => v.as_slice(), other => panic!("expected sum entries, got {:?}", other), diff --git a/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs b/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs index 793953362..39e18ab74 100644 --- a/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs +++ b/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs @@ -743,7 +743,7 @@ mod tests { in_tx .entries .iter() - .map(|(_, key)| key.clone()) + .map(|e| e.primary_key.clone()) .collect::>(), vec![ b"k0000025".to_vec(), diff --git a/grovedb/src/tests/indexed_axis_proof_tests.rs b/grovedb/src/tests/indexed_axis_proof_tests.rs index cfbbed8c0..7ed1b4cbc 100644 --- a/grovedb/src/tests/indexed_axis_proof_tests.rs +++ b/grovedb/src/tests/indexed_axis_proof_tests.rs @@ -18,6 +18,8 @@ mod tests { use grovedb_merk::proofs::{query::QueryItem as MerkQueryItem, Query as MerkQuery}; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntry; + use crate::{ operations::proof::indexed_axis::AxisEntries, tests::{make_test_grovedb, TEST_LEAF}, @@ -111,21 +113,21 @@ mod tests { db.root_hash(None, grove_version).unwrap().expect("root") } - fn entries_as_count(entries: &AxisEntries) -> &[(u64, Vec)] { + fn entries_as_count(entries: &AxisEntries) -> &[IndexedAxisEntry] { match entries { AxisEntries::Count(v) => v.as_slice(), other => panic!("expected count entries, got {:?}", other), } } - fn entries_as_sum(entries: &AxisEntries) -> &[(i64, Vec)] { + fn entries_as_sum(entries: &AxisEntries) -> &[IndexedAxisEntry] { match entries { AxisEntries::Sum(v) => v.as_slice(), other => panic!("expected sum entries, got {:?}", other), } } - fn entries_as_avg(entries: &AxisEntries) -> &[(i128, Vec)] { + fn entries_as_avg(entries: &AxisEntries) -> &[IndexedAxisEntry] { match entries { AxisEntries::Avg(v) => v.as_slice(), other => panic!("expected avg entries, got {:?}", other), @@ -452,8 +454,8 @@ mod tests { // of original_key for ties (b/c/a → c, b, a). let entries = entries_as_count(&result.entries); assert_eq!(entries.len(), 3); - for (c, _) in entries { - assert_eq!(*c, 1); + for entry in entries { + assert_eq!(entry.ordering_value, 1); } assert_eq!(result.root_hash, root_hash(&db, grove_version)); } @@ -602,8 +604,8 @@ mod tests { // Skip-1 then take-2 → c, b. let entries = entries_as_count(&result.entries); assert_eq!(entries.len(), 2); - for (c, _) in entries { - assert_eq!(*c, 1); + for entry in entries { + assert_eq!(entry.ordering_value, 1); } assert_eq!(result.skipped, 1); } @@ -1991,15 +1993,29 @@ mod tests { #[test] fn axis_entries_helpers() { - let c = AxisEntries::Count(vec![(1u64, b"a".to_vec())]); + let entry = |v: u64| IndexedAxisEntry { + ordering_value: v, + primary_key: b"a".to_vec(), + value: Element::new_item(b"v".to_vec()), + }; + let c = AxisEntries::Count(vec![entry(1)]); assert_eq!(c.len(), 1); assert!(!c.is_empty()); let empty_c = AxisEntries::Count(vec![]); assert_eq!(empty_c.len(), 0); assert!(empty_c.is_empty()); - let s = AxisEntries::Sum(vec![(1i64, b"a".to_vec()), (2i64, b"b".to_vec())]); + let sum_entry = |v: i64, k: &[u8]| IndexedAxisEntry { + ordering_value: v, + primary_key: k.to_vec(), + value: Element::new_sum_item(v), + }; + let s = AxisEntries::Sum(vec![sum_entry(1, b"a"), sum_entry(2, b"b")]); assert_eq!(s.len(), 2); - let a = AxisEntries::Avg(vec![(1i128, b"a".to_vec())]); + let a = AxisEntries::Avg(vec![IndexedAxisEntry { + ordering_value: 1i128, + primary_key: b"a".to_vec(), + value: Element::new_item(b"v".to_vec()), + }]); assert_eq!(a.len(), 1); } @@ -2760,9 +2776,9 @@ mod tests { .expect("verify"); let got = entries_as_count(&result.entries); assert_eq!(got.len(), 3); - assert_eq!(got[0].0, 3); - assert_eq!(got[1].0, 2); - assert_eq!(got[2].0, 1); + assert_eq!(got[0].ordering_value, 3); + assert_eq!(got[1].ordering_value, 2); + assert_eq!(got[2].ordering_value, 1); assert_eq!(result.skipped, 6); } @@ -3155,7 +3171,7 @@ mod tests { .expect("verify"); let got = entries_as_count(&result.entries); assert_eq!(got.len(), 1); - assert_eq!(got[0].0, 2); + assert_eq!(got[0].ordering_value, 2); } // ---------- post-mutation / cross-check / scale ---------- @@ -3178,8 +3194,8 @@ mod tests { let got = entries_as_count(&result.entries); assert_eq!(got.len(), 2); // c(3) and a(1) remain. - assert_eq!(got[0].1, b"c".to_vec()); - assert_eq!(got[1].1, b"a".to_vec()); + assert_eq!(got[0].primary_key, b"c".to_vec()); + assert_eq!(got[1].primary_key, b"a".to_vec()); } #[test] @@ -3236,8 +3252,8 @@ mod tests { .expect("verify"); let got = entries_as_count(&result.entries); assert_eq!(got.len(), 10); - assert_eq!(got[0].0, 29); - assert_eq!(got[9].0, 20); + assert_eq!(got[0].ordering_value, 29); + assert_eq!(got[9].ordering_value, 20); } // ---------- triple-nested cidx ---------- @@ -3290,7 +3306,7 @@ mod tests { .expect("verify"); let got = entries_as_count(&result.entries); assert_eq!(got.len(), 4); - assert_eq!(got[0].0, 7); + assert_eq!(got[0].ordering_value, 7); assert_eq!(result.root_hash, root_hash(&db, grove_version)); } diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index d2a03713d..3683e8be1 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -648,6 +648,162 @@ mod tests { ); } + /// A NESTED INDEXED TREE as a primary entry. + /// + /// Its committed value hash is a three-way + /// `combine_hash_three(H(value), primary_root, secondary_root)`, which + /// no single child-hash witness can express. The target-chain + /// commitment enum carries the pieces instead, so this shape proves + /// and verifies like any other rather than being refused. + #[test] + fn a_nested_indexed_tree_primary_proves_and_resolves() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("outer PCIT"); + // A PCIT nested inside a PCIT: the inner element is the primary + // entry whose commitment the outer row must bind. + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"inner", + Element::empty_provable_count_indexed_tree(), + None, + grove_version, + ) + .unwrap() + .expect("inner PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx", b"inner"].as_ref(), + b"leaf", + Element::new_item(b"v".to_vec()), + None, + grove_version, + ) + .unwrap() + .expect("inner entry"); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "nested indexed tree reported: {issues:?}" + ); + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("a nested indexed-tree primary must be provable"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("verify"); + assert_eq!( + result.root_hash, + db.root_hash(None, grove_version).unwrap().unwrap() + ); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].primary_key, b"inner".to_vec()); + assert!( + matches!( + entries[0].value.underlying(), + Element::ProvableCountIndexedTree(..) + ), + "the row must resolve to the nested indexed element itself, got {}", + entries[0].value.type_str() + ); + } + + /// A REFERENCE-SHAPED primary entry resolves through to its terminal + /// on both the direct and the proved path, and the two agree. + /// + /// The row still BINDS the immediate primary node — that is what keeps + /// the mirror's invariant local — while the value handed back is what + /// `db.get` on that key would return. Both halves matter, so both are + /// asserted. + #[test] + fn a_reference_shaped_primary_resolves_to_its_terminal() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_item(b"terminal-value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("terminal"); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"ref", + Element::new_reference(ReferencePathType::UpstreamRootHeightReference( + 1, + vec![b"target".to_vec()], + )), + None, + grove_version, + ) + .unwrap() + .expect("reference-shaped primary"); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "reference-shaped primary reported: {issues:?}" + ); + + let expected = Element::new_item(b"terminal-value".to_vec()); + + // Direct read. + let direct = db + .indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) + .unwrap() + .expect("direct top_k"); + assert_eq!(direct.len(), 1); + assert_eq!( + direct[0].value, expected, + "a direct read must resolve a reference-shaped primary to its terminal" + ); + + // Proved read must agree. + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("prove"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("verify"); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + assert_eq!(entries.len(), 1); + assert_eq!( + entries[0].value, expected, + "a proved read must resolve to the same terminal the direct read returned" + ); + assert_eq!(entries[0].primary_key, b"ref".to_vec()); + } + /// Ordinary user references keep their own semantics. The one-hop /// immediate-node rule is dedicated indexed-tree behaviour, so it must /// not leak into how a normal reference elsewhere in the grove is diff --git a/grovedb/src/tests/indexed_tree_security_regression_tests.rs b/grovedb/src/tests/indexed_tree_security_regression_tests.rs index 97ac00276..ff6aec47a 100644 --- a/grovedb/src/tests/indexed_tree_security_regression_tests.rs +++ b/grovedb/src/tests/indexed_tree_security_regression_tests.rs @@ -488,7 +488,7 @@ fn derived_counts_order_the_secondary_index() { ) .unwrap() .expect("top_k"); - let order: Vec> = top.iter().map(|(_, k)| k.clone()).collect(); + let order: Vec> = top.iter().map(|e| e.primary_key.clone()).collect(); assert_eq!( order, vec![b"b".to_vec(), b"c".to_vec(), b"a".to_vec()], diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 830cdb6bf..64e78a76a 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -75,6 +75,7 @@ mod partial_batch_consistency_tests; mod proof_advanced_tests; mod proof_coverage_tests; mod proof_depth_limit_tests; +mod proof_size_measurement; mod provable_count_indexed_tree_tests; mod provable_count_provable_sum_indexed_tree_tests; mod provable_count_provable_sum_tree_tests; diff --git a/grovedb/src/tests/proof_size_measurement.rs b/grovedb/src/tests/proof_size_measurement.rs new file mode 100644 index 000000000..58b23e70a --- /dev/null +++ b/grovedb/src/tests/proof_size_measurement.rs @@ -0,0 +1,94 @@ +//! Proof-size guard for the reference-backed indexed-axis proofs. +//! +//! The whole point of rows carrying their value is that a top-k proof +//! costs one value plus a hash per row instead of a per-row inclusion +//! proof. That is a property worth pinning: a future change that reaches +//! for per-row path proofs would still be correct, and would silently +//! multiply proof size several-fold. + +#[cfg(test)] +mod tests { + use grovedb_version::version::GroveVersion; + + use crate::{ + tests::{make_test_grovedb, TEST_LEAF}, + Element, GroveDb, + }; + + /// Marginal proof cost per additional returned row must stay small and + /// flat — the signature of "the value rides along with the row". + /// + /// Tree-shaped children on purpose: a count-indexed tree indexes its + /// children's counts, so trees are the normal case, and they are the + /// shape a per-row inclusion proof would be most expensive for. + #[test] + fn top_k_proof_cost_per_row_stays_flat() { + let v = GroveVersion::latest(); + let db = make_test_grovedb(v); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + v, + ) + .unwrap() + .expect("pcit"); + for i in 0..32usize { + let key = format!("k{i:03}"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + key.as_bytes(), + Element::empty_tree(), + None, + v, + ) + .unwrap() + .expect("child"); + for j in 0..(i % 4) { + db.insert( + [TEST_LEAF, b"cidx", key.as_bytes()].as_ref(), + format!("i{j}").as_bytes(), + Element::new_item(vec![b'x'; 8]), + None, + None, + v, + ) + .unwrap() + .expect("grandchild"); + } + } + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let size_for = |k: u16| -> usize { + let p = db + .prove_indexed_count_top_k(path, k, true, None, v) + .unwrap() + .expect("prove"); + let res = GroveDb::verify_indexed_count_top_k(&p, path, k, true, v).expect("verify"); + assert_eq!(res.entries.len(), k as usize, "must return k rows"); + assert_eq!( + res.root_hash, + db.root_hash(None, v).unwrap().unwrap(), + "verified proof must reconstruct the grove root" + ); + p.len() + }; + + let one = size_for(1); + let sixteen = size_for(16); + let per_row = (sixteen - one) / 15; + + // Generous ceiling: the real figure is well under this, and the + // bound exists to catch an order-of-magnitude regression (a per-row + // path proof costs hundreds of bytes), not to pin an exact size. + assert!( + per_row < 200, + "marginal proof cost per returned row regressed to {per_row} bytes \ + (k=1: {one}, k=16: {sixteen}); a reference row should carry its value \ + for roughly the value's size plus a hash, not a per-row inclusion proof" + ); + eprintln!("per-row marginal proof cost: {per_row} bytes (k=1 {one}, k=16 {sixteen})"); + } +} diff --git a/grovedb/src/tests/provable_count_sum_tree_tests.rs b/grovedb/src/tests/provable_count_sum_tree_tests.rs index 986a64849..ac862bf8c 100644 --- a/grovedb/src/tests/provable_count_sum_tree_tests.rs +++ b/grovedb/src/tests/provable_count_sum_tree_tests.rs @@ -103,7 +103,6 @@ mod tests { Node::KVCountSum(k, ..) => k.clone(), Node::KVDigestCountSum(k, ..) => k.clone(), Node::KVRefValueHashCountSum(k, ..) => k.clone(), - Node::KVRefValueHashCountSumWithTargetChildHash(k, ..) => k.clone(), Node::KVHashCount(..) => vec![], Node::Hash(_) | Node::KVHash(_) => vec![], // HashWithCount is keyless (collapsed subtree representation diff --git a/merk/src/merk/chunks.rs b/merk/src/merk/chunks.rs index bc6d23dd7..94fdb30fa 100644 --- a/merk/src/merk/chunks.rs +++ b/merk/src/merk/chunks.rs @@ -487,9 +487,6 @@ mod test { Node::KVCount(..) => counts.kv += 1, Node::KVHashCount(..) => counts.kv_hash += 1, Node::KVRefValueHashCount(..) => counts.kv_ref_value_hash += 1, - Node::KVRefValueHashCountSumWithTargetChildHash(..) => { - counts.kv_ref_value_hash += 1 - } // HashWithCount is hash-equivalent to Hash for the verifier; // count it under `hash` for the test counter. Node::HashWithCount(..) => counts.hash += 1, diff --git a/merk/src/proofs/branch/mod.rs b/merk/src/proofs/branch/mod.rs index 47ef06a0a..91e318d8d 100644 --- a/merk/src/proofs/branch/mod.rs +++ b/merk/src/proofs/branch/mod.rs @@ -125,8 +125,7 @@ impl TrunkQueryResult { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) - | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.clone()), + | Node::KVRefValueHashCountSum(key, ..) => Some(key.clone()), Node::Hash(_) | Node::KVHash(_) | Node::KVHashCount(..) @@ -402,8 +401,7 @@ impl BranchQueryResult { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) - | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.clone()), + | Node::KVRefValueHashCountSum(key, ..) => Some(key.clone()), Node::Hash(_) | Node::KVHash(_) | Node::KVHashCount(..) diff --git a/merk/src/proofs/query/count_offset/verify.rs b/merk/src/proofs/query/count_offset/verify.rs index 902adc63e..f0b42ad17 100644 --- a/merk/src/proofs/query/count_offset/verify.rs +++ b/merk/src/proofs/query/count_offset/verify.rs @@ -198,8 +198,7 @@ pub fn verify_count_offset_on_range_proof( // encoding, so the value bytes are the dereferenced target's // and the node's own hash field is the reference element's. | Node::KVRefValueHashCount(_, _, _, _) - | Node::KVRefValueHashCountSum(_, _, _, _, _) - | Node::KVRefValueHashCountSumWithTargetChildHash(_, _, _, _, _, _) => Ok(()), + | Node::KVRefValueHashCountSum(_, _, _, _, _) => Ok(()), other => Err(Error::InvalidProofError(format!( "unexpected node type in count-offset proof: {}", other @@ -275,7 +274,6 @@ fn aggregate_of_proof_tree_node(tree: &ProofTree) -> Result { // conceptual position as the value-bearing variants. Node::KVRefValueHashCount(_, _, _, c) => Ok(*c), Node::KVRefValueHashCountSum(_, _, _, c, _) => Ok(*c), - Node::KVRefValueHashCountSumWithTargetChildHash(_, _, _, c, _, _) => Ok(*c), Node::KVValueHashFeatureType(_, _, _, ft) => match ft { TreeFeatureType::ProvableCountedMerkNode(c) => Ok(*c), TreeFeatureType::ProvableCountedSummedMerkNode(c, _) => Ok(*c), @@ -433,7 +431,6 @@ fn verify_count_offset_shape( // Resolved-reference per-element variants. Node::KVRefValueHashCount(key, _, _, _) => key.as_slice(), Node::KVRefValueHashCountSum(key, _, _, _, _) => key.as_slice(), - Node::KVRefValueHashCountSumWithTargetChildHash(key, _, _, _, _, _) => key.as_slice(), // Reaching here would require: // - the `execute_with_options` allowlist accepted a node // that doesn't carry a key (only `HashWithCount` / @@ -785,44 +782,6 @@ fn classify_self<'a>( reference_element_hash: Some(*reference_element_hash), }) } - Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - reference_element_hash, - _, - _, - target_child_hash, - ) => { - if !in_range { - return Err(Error::InvalidProofError( - "count-offset proof: KVRefValueHashCountSumWithTargetChildHash at an \ - out-of-range position" - .to_string(), - )); - } - if own_count != 1 { - return Err(Error::InvalidProofError(format!( - "count-offset proof: KVRefValueHashCountSumWithTargetChildHash at \ - own_count={} (expected 1)", - own_count - ))); - } - // Two combines, mirroring `Tree::hash` for this variant: the - // target is layered, so its committed hash folds its child - // commitment in before the reference folds over it. - let target_committed = crate::tree::combine_hash( - &compute_value_hash(value.as_slice()).unwrap(), - target_child_hash, - ) - .unwrap(); - Ok(BoundaryKind::ValueReturned { - key: key.as_slice(), - value: value.as_slice(), - value_hash: crate::tree::combine_hash(reference_element_hash, &target_committed) - .unwrap(), - reference_element_hash: Some(*reference_element_hash), - }) - } Node::KVValueHash(key, value, _) => { // Non-count fallback. Only legitimate if the prover hit a // raw / unknown element type and fell back to the regular diff --git a/merk/src/proofs/query/verify.rs b/merk/src/proofs/query/verify.rs index 6f26e2706..e5e4a92c9 100644 --- a/merk/src/proofs/query/verify.rs +++ b/merk/src/proofs/query/verify.rs @@ -221,8 +221,6 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} - Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} - Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // cannot verify lower bound - we have an abridged // tree, so we cannot tell what the preceding key was @@ -265,8 +263,6 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} - Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} - Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // cannot verify upper bound - we have an abridged // tree so we cannot tell what the previous key was @@ -595,26 +591,6 @@ impl QueryProofVerify for Query { } execute_node(key, Some(value), *value_hash, false)?; } - Node::KVRefValueHashCountSumWithTargetChildHash( - key, - value, - value_hash, - _count, - _sum, - _target_child_hash, - ) => { - #[cfg(feature = "proof_debug")] - { - println!("Processing KVRefValueHashCountSumWithTargetChildHash node"); - } - // `child_hash_verified = true`: unlike the plain ref - // variant, the hash reconstruction for this node folds - // the target's child commitment in explicitly (see - // `Tree::hash`), so by the time the root matches, the - // target's layered binding has been independently - // checked — which is exactly what the flag means. - execute_node(key, Some(value), *value_hash, true)?; - } } last_push = Some(node.clone()); @@ -658,7 +634,6 @@ impl QueryProofVerify for Query { Some(Node::KVCountSum(..)) => {} Some(Node::KVDigestCountSum(..)) => {} Some(Node::KVRefValueHashCountSum(..)) => {} - Some(Node::KVRefValueHashCountSumWithTargetChildHash(..)) => {} // proof contains abridged data so we cannot verify absence of // remaining query items diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index bedf0c3c4..bf65dd62a 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -364,39 +364,6 @@ impl Tree { ) }) } - Node::KVRefValueHashCountSumWithTargetChildHash( - key, - referenced_value, - node_value_hash, - count, - sum, - target_child_hash, - ) => { - // Two combines rather than one: the target is layered, so - // its own committed value hash is - // `combine_hash(H(target), target_child_hash)` — not - // `H(target)`. Reconstructing that first, then folding the - // reference over it, is what binds BOTH the target's bytes - // and its child commitment into the row's hash. Substituting - // either one breaks the root. - let mut cost = OperationCost::default(); - let referenced_value_hash = - value_hash(referenced_value.as_slice()).unwrap_add_cost(&mut cost); - let target_committed_hash = combine_hash(&referenced_value_hash, target_child_hash) - .unwrap_add_cost(&mut cost); - let combined_value_hash = combine_hash(node_value_hash, &target_committed_hash) - .unwrap_add_cost(&mut cost); - - kv_digest_to_kv_hash(key.as_slice(), &combined_value_hash).flat_map(|kv_hash| { - node_hash_with_count_and_sum( - &kv_hash, - &self.child_hash(true), - &self.child_hash(false), - *count, - *sum, - ) - }) - } } } @@ -574,8 +541,7 @@ impl Tree { | Node::KVRefValueHashSum(key, ..) | Node::KVCountSum(key, ..) | Node::KVDigestCountSum(key, ..) - | Node::KVRefValueHashCountSum(key, ..) - | Node::KVRefValueHashCountSumWithTargetChildHash(key, ..) => Some(key.as_slice()), + | Node::KVRefValueHashCountSum(key, ..) => Some(key.as_slice()), // These nodes don't have keys, only hashes Node::Hash(_) | Node::KVHash(_) From f68574efeb86c667d5b39c3aad681855d96e5c6c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 05:13:17 +0700 Subject: [PATCH 05/18] fix: correct reference-chain semantics; adopt #816's API and lint hygiene (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's review of #816 vs #817 identified two real correctness defects in this branch's target chains. Both are confirmed, reproduced by new tests, and fixed. It also flagged three hygiene items worth adopting. **Defect 1 — multi-hop chains folded the wrong hash.** A GroveDB reference commits its TERMINAL's value hash, not the next hop's: `follow_reference_get_value_hash` recurses past every intermediate reference before the hash reaches `PutCombinedReference` (batch/mod.rs:2176). The chain fold composed hop-by-hop, which happens to agree at one hop and diverges at two, so the existing one-hop test could not catch it. A two-hop primary failed verification with a spurious "bound to a different primary commitment". The chain is now at most TWO entries — head, and the terminal when the head is a reference. Intermediate hops are not carried at all, because nothing binds them: the head commits the terminal directly, so carrying the middle would hand a verifier bytes it cannot check. **Defect 2 — relative references resolved against the wrong path.** `SiblingReference` appends its key to the path it is given, so that path must be the entry's PARENT. Both the chain builder and the direct-read resolver passed parent‖key, one segment too deep, sending resolution underneath the entry itself. An `UpstreamRootHeightReference` masks this (it truncates to the first N segments and lands in the same place), which is why the existing test passed. A sibling-reference primary failed at prove time with "parent exists but is not a tree". Both now have dedicated tests asserting direct and proved reads agree. **Adopted from #816:** - **Removed the `PartialEq<(T, Vec)>` shim.** Codex is right that an equality impl silently ignoring `value` lets an assertion keep passing while resolution returns the wrong element. Replaced with an explicit `IndexedAxisEntrySliceExt::key_pairs()` projection, so each call site says which half it compares — and ranking-only callers get a real API instead of a comparison trick. 107 assertions migrated. - **`primary_unreachable_node` / `secondary_unreachable_node` sentinels.** A raw-iterated node the AVL cannot reach is corruption with its own name; silently skipping its commitment check made an operator guess. - **`cargo clippy -D warnings` clean** on grovedb and grovedb-merk. Also removed `CountOffsetReturnedItem::reference_element_hash`, which the chain redesign left set but never read. Full workspace suite passes (43 targets, 2739 grovedb tests). Per-row marginal proof cost unchanged at 83 bytes. --- grovedb/src/batch/indexed_tree/mod.rs | 2 +- grovedb/src/lib.rs | 32 +- grovedb/src/operations/indexed_tree.rs | 10 +- grovedb/src/operations/mmr_tree.rs | 9 +- .../operations/proof/indexed_axis/envelope.rs | 25 +- .../proof/indexed_axis/target_chain.rs | 304 ++++++++++++------ grovedb/src/query_result_type.rs | 35 +- .../tests/batch_indexed_fresh_create_tests.rs | 14 +- .../tests/batch_indexed_multi_axis_tests.rs | 48 ++- .../tests/batch_indexed_overwrite_tests.rs | 8 +- .../src/tests/coverage_batch_indexed_tests.rs | 8 +- grovedb/src/tests/coverage_round7_tests.rs | 7 +- .../indexed_axis_nested_and_bounds_tests.rs | 10 +- .../tests/indexed_axis_offset_proof_tests.rs | 21 +- .../indexed_axis_paginated_cost_tests.rs | 8 +- grovedb/src/tests/indexed_axis_proof_tests.rs | 51 +-- .../src/tests/indexed_reference_row_tests.rs | 164 ++++++++++ .../indexed_tree_secondary_drift_tests.rs | 37 ++- .../indexed_tree_security_regression_tests.rs | 13 +- .../provable_count_indexed_tree_tests.rs | 21 +- ...e_count_provable_sum_indexed_tree_tests.rs | 34 +- .../tests/provable_sum_indexed_tree_tests.rs | 23 +- .../src/tests/verify_grovedb_indexed_tests.rs | 6 +- merk/src/proofs/query/count_offset/tests.rs | 2 - merk/src/proofs/query/count_offset/verify.rs | 24 -- 25 files changed, 630 insertions(+), 286 deletions(-) diff --git a/grovedb/src/batch/indexed_tree/mod.rs b/grovedb/src/batch/indexed_tree/mod.rs index 4c5fee5a6..4b623a345 100644 --- a/grovedb/src/batch/indexed_tree/mod.rs +++ b/grovedb/src/batch/indexed_tree/mod.rs @@ -55,7 +55,7 @@ pub(crate) use delete_tree::validate_delete_tree_type; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{element::costs::ElementCostExtensions, CryptoHash, Merk}; +use grovedb_merk::{element::costs::ElementCostExtensions, Merk}; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; pub(crate) use mirror::{apply_indexed_secondary_mirror_post_apply, read_post_apply_transitions}; diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 95dd7a0c7..e8dca4fbc 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -260,7 +260,7 @@ pub use query::{ PathTrunkChunkQuery, SizedQuery, }; #[cfg(any(feature = "minimal", feature = "verify"))] -pub use query_result_type::IndexedAxisEntry; +pub use query_result_type::{IndexedAxisEntry, IndexedAxisEntrySliceExt}; #[cfg(feature = "minimal")] use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] @@ -1000,7 +1000,7 @@ impl GroveDb { pub(crate) fn propagate_changes_with_transaction_with_initial_deferred<'b, B: AsRef<[u8]>>( &self, - mut merk_cache: HashMap, Merk>, + merk_cache: HashMap, Merk>, path: SubtreePath<'b, B>, initial_deferred_secondary: Option<(Hash, Option>)>, transaction: &Transaction, @@ -2057,11 +2057,11 @@ impl GroveDb { // `value_hash(bytes)` cannot reproduce. // // `None` means the raw iterator found bytes the AVL cannot reach - // from the committed root — an unlinked row. That is corruption in - // its own right, and it is reported by the shape checks below; we - // just cannot state a commitment for it, so the commitment - // comparison is skipped rather than turned into a hard error that - // would mask every other finding in the walk. + // from the committed root — an unlinked node. That is corruption + // in its own right and gets its own sentinel: an operator seeing + // a stale-commitment report should not have to guess whether the + // node was reachable at all. The walk continues so the rest of + // the relational report still lands. let target_value_hash = primary_merk .get_value_hash( &p_key, @@ -2070,6 +2070,12 @@ impl GroveDb { grove_version, ) .unwrap()?; + if target_value_hash.is_none() { + let mut p = new_path.to_vec(); + p.push(sentinel("primary_unreachable_node")); + p.push(p_key.clone()); + issues.insert(p, ([0u8; 32], [0u8; 32], [0u8; 32])); + } expected.insert( p_key.clone(), ( @@ -2103,10 +2109,8 @@ impl GroveDb { // it is what proves the row still binds the primary entry's // current state. Recomputing it from `sec_value` alone would only // re-derive the reference bytes' own hash and see nothing stale. - // `None` here means the row is present in storage but unreachable - // from the committed root — an unlinked row, which the shape and - // orphan checks below already surface. Skip its commitment - // comparison rather than hard-erroring the whole walk. + // Same as the primary side: an unlinked row is corruption with its + // own name, not a silent skip. let stored_value_hash = secondary_merk .get_value_hash( &sec_key, @@ -2115,6 +2119,12 @@ impl GroveDb { grove_version, ) .unwrap()?; + if stored_value_hash.is_none() { + let mut p = new_path.to_vec(); + p.push(sentinel("secondary_unreachable_node")); + p.push(sec_key.clone()); + issues.insert(p, ([0u8; 32], [0u8; 32], [0u8; 32])); + } let item_key = sec_key[sort_len..].to_vec(); actual.entry(item_key).or_default().push(( sec_key.clone(), diff --git a/grovedb/src/operations/indexed_tree.rs b/grovedb/src/operations/indexed_tree.rs index 44ed9d598..c6579e523 100644 --- a/grovedb/src/operations/indexed_tree.rs +++ b/grovedb/src/operations/indexed_tree.rs @@ -2919,11 +2919,15 @@ where let value = match element.underlying() { Element::Reference(reference_path, ..) | Element::ReferenceWithSumItem(reference_path, ..) => { - let mut qualified = indexed_path.to_vec(); - qualified.push(primary_key.clone()); + // The entry's PARENT path, not its own qualified path: a + // relative reference resolves against the parent + // (`SiblingReference` appends its key to what it is + // given), so passing the entry's own path would look for a + // child underneath the entry itself. + let parent_path = indexed_path.to_vec(); let absolute = match crate::reference_path::path_from_reference_path_type( reference_path.clone(), - &qualified, + &parent_path, Some(primary_key.as_slice()), ) { Ok(p) => p, diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index e725fe554..1fd8dd990 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -9,13 +9,8 @@ use std::collections::HashMap; -use grovedb_costs::{ - cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, -}; -use grovedb_merk::element::{ - costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, - insert::ElementInsertToStorageExtensions, -}; +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_merk::element::insert::ElementInsertToStorageExtensions; use grovedb_merkle_mountain_range::{ hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, }; diff --git a/grovedb/src/operations/proof/indexed_axis/envelope.rs b/grovedb/src/operations/proof/indexed_axis/envelope.rs index f7cc5f828..2227094bc 100644 --- a/grovedb/src/operations/proof/indexed_axis/envelope.rs +++ b/grovedb/src/operations/proof/indexed_axis/envelope.rs @@ -73,9 +73,13 @@ pub enum IndexedTargetCommitment { axes: Vec<(u8, [u8; 32])>, }, /// Reference node, committed as - /// `combine_hash(H(value), next_node_commitment)` — the next hop's - /// commitment is reconstructed from the following chain entry rather - /// than carried, which is what keeps a chain self-authenticating. + /// `combine_hash(H(value), terminal_commitment)`. + /// + /// Note the TERMINAL, not the next hop: `follow_reference_get_value_hash` + /// recurses past every intermediate reference before the hash is baked + /// into `PutCombinedReference`, so a reference three hops from its + /// terminal still commits that terminal's hash directly. Folding + /// hop-by-hop instead happens to agree at one hop and diverges at two. Reference, } @@ -89,8 +93,16 @@ pub struct IndexedTargetNode { pub commitment: IndexedTargetCommitment, } +impl IndexedTargetNode { + /// Whether this node is a reference (and so commits its terminal's + /// hash rather than its own bytes' hash). + pub fn is_reference(&self) -> bool { + matches!(self.commitment, IndexedTargetCommitment::Reference) + } +} + /// The resolved target of one secondary row: the immediate primary node, -/// followed by any ordinary reference hops through to the terminal. +/// and — only when that node is a reference — the TERMINAL it resolves to. /// /// **No per-node path proofs.** A chain authenticates itself from the /// row's own committed value hash: each entry's commitment is @@ -109,7 +121,10 @@ pub struct IndexedTargetNode { /// instead of `k` inclusion proofs. #[derive(Encode, Decode, Debug, Clone, PartialEq, Eq)] pub struct IndexedTargetChain { - /// Immediate primary node first, terminal last. Never empty. + /// One entry when the primary entry is directly valued; two — head + /// then terminal — when it is a reference. Intermediate hops are not + /// carried: nothing binds them, since the head commits the terminal + /// directly. pub nodes: Vec, } diff --git a/grovedb/src/operations/proof/indexed_axis/target_chain.rs b/grovedb/src/operations/proof/indexed_axis/target_chain.rs index ddc74ee2c..f0ffe0745 100644 --- a/grovedb/src/operations/proof/indexed_axis/target_chain.rs +++ b/grovedb/src/operations/proof/indexed_axis/target_chain.rs @@ -161,8 +161,17 @@ fn build_commitment<'db>( Ok(commitment).wrap_with_cost(cost) } -/// Build the chain for one secondary row: the immediate primary entry, -/// then any ordinary reference hops through to the terminal. +/// Build the chain for one secondary row. +/// +/// The chain is at most TWO entries: the immediate primary entry, and — +/// only when that entry is itself a reference — the TERMINAL it resolves +/// to. Intermediate hops are deliberately not carried, because nothing +/// binds them: a GroveDB reference commits its terminal's value hash +/// directly (`follow_reference_get_value_hash` recurses past every +/// intermediate reference before the hash is baked into +/// `PutCombinedReference`), so the head's commitment reaches the terminal +/// in one step no matter how many hops the path takes. Carrying the +/// middle of the chain would hand a verifier bytes it cannot check. pub(crate) fn build_target_chain<'db>( grovedb: &'db GroveDb, indexed_path: &[Vec], @@ -172,93 +181,171 @@ pub(crate) fn build_target_chain<'db>( grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); - let mut qualified_path = indexed_path.to_vec(); - qualified_path.push(primary_key.to_vec()); + + // The head: the primary entry itself. + let head_parent: Vec<&[u8]> = indexed_path.iter().map(Vec::as_slice).collect(); + let head_merk = cost_return_on_error!( + &mut cost, + grovedb.open_transactional_merk_at_path( + head_parent.as_slice().into(), + transaction, + Some(batch), + grove_version, + ) + ); + let head_element = cost_return_on_error!( + &mut cost, + Element::get(&head_merk, primary_key, true, grove_version).map_err(|e| { + Error::CorruptedData(format!( + "indexed target chain: primary entry {} is missing: {e}", + hex::encode(primary_key) + )) + }) + ); + drop(head_merk); + + let mut head_qualified = indexed_path.to_vec(); + head_qualified.push(primary_key.to_vec()); + let head = cost_return_on_error!( + &mut cost, + build_chain_node( + grovedb, + &head_element, + &head_qualified, + transaction, + batch, + grove_version, + ) + ); + let mut nodes = vec![head]; + + // If the primary entry is a reference, walk to the terminal. Only the + // terminal is recorded — see this function's doc for why. + let mut current_element = head_element; + let mut current_parent = indexed_path.to_vec(); + let mut current_key = primary_key.to_vec(); let mut visited = std::collections::HashSet::new(); - let mut nodes = Vec::new(); + visited.insert(head_qualified); for _ in 0..=MAX_REFERENCE_HOPS { - if !visited.insert(qualified_path.clone()) { + let reference_path = match current_element.underlying() { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => reference_path.clone(), + // Not a reference: the head was the terminal, or we just + // reached it. + _ => { + if nodes.len() > 1 || !nodes[0].is_reference() { + return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); + } + // Unreachable: a reference head always takes the branch + // below at least once before we get here. + return Err(Error::CorruptedCodeExecution( + "indexed target chain: reference head produced no terminal", + )) + .wrap_with_cost(cost); + } + }; + + // `current_parent` — NOT the node's own qualified path. A relative + // reference resolves against its PARENT (`SiblingReference` + // appends its key to what it is given), so passing the node's own + // path would look for a child underneath the entry itself. + let next_qualified = match path_from_reference_path_type( + reference_path, + ¤t_parent, + Some(current_key.as_slice()), + ) { + Ok(p) => p, + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + }; + if !visited.insert(next_qualified.clone()) { return Err(Error::CyclicReference).wrap_with_cost(cost); } - let Some((key, parent_segments)) = qualified_path.split_last() else { + let Some((next_key, next_parent)) = next_qualified.split_last() else { return Err(Error::CorruptedPath( "indexed target chain resolved an empty path".to_string(), )) .wrap_with_cost(cost); }; - let parent_slices: Vec<&[u8]> = parent_segments.iter().map(Vec::as_slice).collect(); - let parent_merk = cost_return_on_error!( + let next_parent_slices: Vec<&[u8]> = next_parent.iter().map(Vec::as_slice).collect(); + let next_merk = cost_return_on_error!( &mut cost, grovedb.open_transactional_merk_at_path( - parent_slices.as_slice().into(), + next_parent_slices.as_slice().into(), transaction, Some(batch), grove_version, ) ); - let element = cost_return_on_error!( + let next_element = cost_return_on_error!( &mut cost, - Element::get(&parent_merk, key, true, grove_version).map_err(|e| { - Error::CorruptedData(format!( - "indexed target chain: entry {} is missing: {e}", - hex::encode(key) + Element::get(&next_merk, next_key, true, grove_version).map_err(|e| { + Error::CorruptedReferencePathKeyNotFound(format!( + "indexed target chain: reference target {} is missing: {e}", + hex::encode(next_key) )) }) ); - drop(parent_merk); - - let value = cost_return_on_error_no_add!( - cost, - element.serialize(grove_version).map_err(|e| { - Error::CorruptedData(format!("indexed target chain: serializing entry: {e}")) - }) - ); - let commitment = cost_return_on_error!( - &mut cost, - build_commitment( - grovedb, - &element, - &qualified_path, - transaction, - batch, - grove_version, - ) - ); - - let next_path = match element.underlying() { - Element::Reference(reference_path, ..) - | Element::ReferenceWithSumItem(reference_path, ..) => { - let mut current = parent_segments.to_vec(); - current.push(key.clone()); - match path_from_reference_path_type( - reference_path.clone(), - ¤t, - Some(key.as_slice()), - ) { - Ok(p) => Some(p), - Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), - } - } - _ => None, - }; + drop(next_merk); - nodes.push(IndexedTargetNode { value, commitment }); - match next_path { - Some(next) => qualified_path = next, - None => return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost), + if !next_element.underlying().is_reference() { + let terminal = cost_return_on_error!( + &mut cost, + build_chain_node( + grovedb, + &next_element, + &next_qualified, + transaction, + batch, + grove_version, + ) + ); + nodes.push(terminal); + return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); } + + current_parent = next_parent.to_vec(); + current_key = next_key.clone(); + current_element = next_element; } Err(Error::ReferenceLimit).wrap_with_cost(cost) } -/// Recompute the committed value hash of one chain entry, given the -/// commitment of the entry that follows it. -fn node_commitment( - node: &IndexedTargetNode, - next: Option<&CryptoHash>, -) -> Result { +/// Serialize one element and derive its commitment shape. +fn build_chain_node<'db>( + grovedb: &'db GroveDb, + element: &Element, + qualified_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, +) -> CostResult { + let mut cost = OperationCost::default(); + let value = cost_return_on_error_no_add!( + cost, + element.serialize(grove_version).map_err(|e| { + Error::CorruptedData(format!("indexed target chain: serializing entry: {e}")) + }) + ); + let commitment = cost_return_on_error!( + &mut cost, + build_commitment( + grovedb, + element, + qualified_path, + transaction, + batch, + grove_version, + ) + ); + Ok(IndexedTargetNode { value, commitment }).wrap_with_cost(cost) +} + +/// The commitment a node's own shape implies, for every shape except +/// `Reference` (whose commitment needs the terminal and is folded by the +/// caller). +fn shape_commitment(node: &IndexedTargetNode) -> Result { let serialized_hash = value_hash(&node.value).value().to_owned(); Ok(match &node.commitment { IndexedTargetCommitment::Simple => serialized_hash, @@ -281,14 +368,9 @@ fn node_commitment( .to_owned() } IndexedTargetCommitment::Reference => { - let next = next.ok_or_else(|| { - Error::CorruptedData( - "indexed target chain: a Reference node is the last entry — a reference \ - commits its target's hash, so the chain must continue" - .to_string(), - ) - })?; - combine_hash(&serialized_hash, next).value().to_owned() + return Err(Error::CorruptedCodeExecution( + "indexed target chain: a Reference node has no standalone shape commitment", + )); } }) } @@ -310,45 +392,59 @@ pub(crate) fn authenticate_target_chain( .to_string(), )); } - if chain.nodes.len() > MAX_REFERENCE_HOPS + 1 { - return Err(Error::CorruptedData(format!( - "indexed target chain has {} entries, above the {MAX_REFERENCE_HOPS}-hop limit", - chain.nodes.len() - ))); - } - // Only the last entry may be a non-reference, and only a reference may - // be followed — otherwise a prover could append unbound entries after - // a terminal and pass off the last one as the resolved value. - for (i, node) in chain.nodes.iter().enumerate() { - let is_last = i + 1 == chain.nodes.len(); - let is_reference = matches!(node.commitment, IndexedTargetCommitment::Reference); - if is_reference == is_last { - return Err(Error::CorruptedData(format!( - "indexed target chain entry {i} is {} but {} the last — a chain is a run of \ - references ending in exactly one non-reference terminal", - if is_reference { - "a reference" - } else { - "not a reference" - }, - if is_last { "is" } else { "is not" } - ))); - } - } - - // Fold back-to-front: each entry's commitment needs the next one's. - let mut next: Option = None; - for node in chain.nodes.iter().rev() { - next = Some(node_commitment(node, next.as_ref())?); - } - let immediate_commitment = next.expect("chain is non-empty"); - let decode = |bytes: &[u8]| -> Result { Element::deserialize(bytes, grove_version).map_err(|e| { Error::CorruptedData(format!("indexed target chain: undecodable element: {e}")) }) }; - let immediate = decode(&chain.nodes[0].value)?; - let terminal = decode(&chain.nodes[chain.nodes.len() - 1].value)?; + let head = &chain.nodes[0]; + + let (immediate_commitment, terminal) = match chain.nodes.len() { + 1 => { + if head.is_reference() { + return Err(Error::CorruptedData( + "indexed target chain: a reference head carries no terminal — a reference \ + commits its terminal's hash, so the terminal is required to rebuild it" + .to_string(), + )); + } + (shape_commitment(head)?, decode(&head.value)?) + } + 2 => { + let terminal_node = &chain.nodes[1]; + if !head.is_reference() { + return Err(Error::CorruptedData( + "indexed target chain: a directly-valued head carries a terminal — only a \ + reference resolves onward" + .to_string(), + )); + } + if terminal_node.is_reference() { + return Err(Error::CorruptedData( + "indexed target chain: the terminal entry is itself a reference — the \ + chain must end at a directly-valued element" + .to_string(), + )); + } + // The head commits the TERMINAL's commitment directly, however + // many hops the reference path actually takes. + let terminal_commitment = shape_commitment(terminal_node)?; + let head_hash = value_hash(&head.value).value().to_owned(); + ( + combine_hash(&head_hash, &terminal_commitment) + .value() + .to_owned(), + decode(&terminal_node.value)?, + ) + } + n => { + return Err(Error::CorruptedData(format!( + "indexed target chain has {n} entries; a chain is either a directly-valued \ + primary or a reference plus its terminal" + ))); + } + }; + + let immediate = decode(&head.value)?; Ok((immediate, immediate_commitment, terminal)) } diff --git a/grovedb/src/query_result_type.rs b/grovedb/src/query_result_type.rs index 76334071a..f63ffcc56 100644 --- a/grovedb/src/query_result_type.rs +++ b/grovedb/src/query_result_type.rs @@ -37,33 +37,30 @@ pub struct IndexedAxisEntry { impl IndexedAxisEntry { /// The `(ordering_value, primary_key)` pair, dropping the value. /// - /// For callers that genuinely only rank — leaderboards, ranking views - /// — and for comparing against the pre-resolution shape. + /// For callers that genuinely only rank — leaderboards, ranking views. pub fn key_pair(self) -> (T, Vec) { (self.ordering_value, self.primary_key) } } -/// Compare an entry against a bare `(ordering_value, primary_key)` pair. +/// Project a page of entries down to `(ordering_value, primary_key)`. /// -/// A migration and ranking-assertion convenience: it deliberately ignores -/// `value`, so it answers "is this row in the right place?" and NOT "does -/// it carry the right value". Tests that care about resolved values must -/// assert on `value` directly — comparing against a tuple will not do it -/// for them. -/// -/// This cannot be confused with ordinary equality: `IndexedAxisEntry`'s -/// own derived `PartialEq` compares every field, and this impl is only -/// reachable when one side is explicitly a tuple. -impl PartialEq<(T, Vec)> for IndexedAxisEntry { - fn eq(&self, other: &(T, Vec)) -> bool { - self.ordering_value == other.0 && self.primary_key == other.1 - } +/// The ranking half of an indexed read, for callers that do not need the +/// values. It is deliberately an explicit projection rather than a +/// cross-type `PartialEq`: an equality impl that quietly ignored `value` +/// would let an assertion keep passing while resolution returned the +/// wrong element, and a caller could not see from the call site which +/// half was being compared. +pub trait IndexedAxisEntrySliceExt { + /// The `(ordering_value, primary_key)` pairs, in order. + fn key_pairs(&self) -> Vec<(T, Vec)>; } -impl PartialEq> for (T, Vec) { - fn eq(&self, other: &IndexedAxisEntry) -> bool { - self.0 == other.ordering_value && self.1 == other.primary_key +impl IndexedAxisEntrySliceExt for [IndexedAxisEntry] { + fn key_pairs(&self) -> Vec<(T, Vec)> { + self.iter() + .map(|e| (e.ordering_value.clone(), e.primary_key.clone())) + .collect() } } diff --git a/grovedb/src/tests/batch_indexed_fresh_create_tests.rs b/grovedb/src/tests/batch_indexed_fresh_create_tests.rs index 28c664fe3..55dbc0170 100644 --- a/grovedb/src/tests/batch_indexed_fresh_create_tests.rs +++ b/grovedb/src/tests/batch_indexed_fresh_create_tests.rs @@ -21,6 +21,8 @@ mod tests { use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::IndexedAxisEntry; use crate::{ @@ -102,7 +104,8 @@ mod tests { assert_eq!( one.indexed_sum_top_k([TEST_LEAF, b"psit"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(7i64, b"a".to_vec()), (-3i64, b"b".to_vec())], "sum index must order the rows inserted alongside the creation" ); @@ -224,7 +227,8 @@ mod tests { assert_eq!( one.indexed_count_top_k([TEST_LEAF, b"t", b"cidx"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"row".to_vec())], ); assert_clean(&one, gv); @@ -269,12 +273,12 @@ mod tests { .expect("fresh inner + row under existing outer"); assert_eq!( - one_key_count(&db, &[TEST_LEAF, b"outer"], gv), + one_key_count(&db, &[TEST_LEAF, b"outer"], gv).key_pairs(), vec![(1u64, b"inner".to_vec())], "the outer index must reflect the fresh inner's derived count of 1" ); assert_eq!( - one_key_count(&db, &[TEST_LEAF, b"outer", b"inner"], gv), + one_key_count(&db, &[TEST_LEAF, b"outer", b"inner"], gv).key_pairs(), vec![(1u64, b"row".to_vec())], "the fresh inner's own index must hold its row" ); @@ -355,7 +359,7 @@ mod tests { .expect("fresh indexed + child tree + grandchild in one batch"); assert_eq!( - one_key_count(&db, &[TEST_LEAF, b"cidx"], gv), + one_key_count(&db, &[TEST_LEAF, b"cidx"], gv).key_pairs(), vec![(1u64, b"child".to_vec())], "the index must record the child's PROPAGATED count of 1" ); diff --git a/grovedb/src/tests/batch_indexed_multi_axis_tests.rs b/grovedb/src/tests/batch_indexed_multi_axis_tests.rs index d0966020d..1b606453a 100644 --- a/grovedb/src/tests/batch_indexed_multi_axis_tests.rs +++ b/grovedb/src/tests/batch_indexed_multi_axis_tests.rs @@ -23,6 +23,8 @@ mod tests { use grovedb_element::indexed::IndexAxis; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, tests::{make_test_grovedb, TempGroveDb, TEST_LEAF}, @@ -251,7 +253,7 @@ mod tests { .unwrap() .expect("sum top_k"); assert_eq!( - by_sum, + by_sum.key_pairs(), vec![ (30, b"a".to_vec()), (20, b"c".to_vec()), @@ -267,7 +269,7 @@ mod tests { .unwrap() .expect("count top_k"); assert_eq!( - by_count, + by_count.key_pairs(), vec![(1, b"a".to_vec()), (1, b"b".to_vec()), (1, b"c".to_vec())], "the count axis must be populated too, not just the sum axis" ); @@ -311,7 +313,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k(path.as_ref(), 10, true, None, gv) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(42, b"a".to_vec())], "the configured sum axis must be populated" ); @@ -372,14 +375,16 @@ mod tests { assert_eq!( db.indexed_sum_top_k(path.as_ref(), 10, true, None, gv) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(99, b"a".to_vec())], "the sum axis must reflect the new sum, with no stale row left behind" ); assert_eq!( db.indexed_count_top_k(path.as_ref(), 10, true, None, gv) .unwrap() - .expect("count top_k"), + .expect("count top_k") + .key_pairs(), vec![(1, b"a".to_vec())], "the count axis must still hold exactly one row for the entry" ); @@ -513,7 +518,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"psit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(3, b"a".to_vec())], ); assert_clean(&db, gv); @@ -566,13 +572,15 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"psit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("psit"), + .expect("psit") + .key_pairs(), vec![(12, b"y".to_vec())] ); assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"idx"].as_ref(), 10, true, None, gv) .unwrap() - .expect("pcpsit sum"), + .expect("pcpsit sum") + .key_pairs(), vec![(6, b"z".to_vec())] ); assert_clean(&db, gv); @@ -605,7 +613,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"idx"].as_ref(), 10, false, None, gv) .unwrap() - .expect("ascending sum top_k"), + .expect("ascending sum top_k") + .key_pairs(), vec![ (-50, b"neg".to_vec()), (0, b"zero".to_vec()), @@ -671,14 +680,16 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"mid", b"idx"].as_ref(), 10, true, None, gv) .unwrap() - .expect("nested sum top_k"), + .expect("nested sum top_k") + .key_pairs(), vec![(15, b"a".to_vec())], "the nested PCPSIT's sum axis must be mirrored through the intermediate level" ); assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"mid", b"idx"].as_ref(), 10, true, None, gv) .unwrap() - .expect("nested count top_k"), + .expect("nested count top_k") + .key_pairs(), vec![(1, b"a".to_vec())], ); assert_clean(&db, gv); @@ -810,7 +821,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"idx"].as_ref(), 10, true, None, gv) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(21, b"a".to_vec())], "an InsertIfNotExists op must mirror like an insert — it carries an element too" ); @@ -1258,7 +1270,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"idx"].as_ref(), 5, true, None, gv) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(10, b"k".to_vec())], "the sum axis must reflect the new total" ); @@ -1453,7 +1466,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(0u64, b"child".to_vec())], "baseline: an empty child is indexed at count 0" ); @@ -1471,7 +1485,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"child".to_vec())], "the generic path must have carried the new aggregate into the index" ); @@ -1492,7 +1507,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(2u64, b"child".to_vec())], "the batch path must maintain the index identically" ); diff --git a/grovedb/src/tests/batch_indexed_overwrite_tests.rs b/grovedb/src/tests/batch_indexed_overwrite_tests.rs index f0d355ad1..683aceca7 100644 --- a/grovedb/src/tests/batch_indexed_overwrite_tests.rs +++ b/grovedb/src/tests/batch_indexed_overwrite_tests.rs @@ -35,6 +35,8 @@ mod tests { use grovedb_element::indexed::IndexAxis; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::{BatchApplyOptions, QualifiedGroveDbOp}, tests::{make_test_grovedb, TempGroveDb, TEST_LEAF}, @@ -149,7 +151,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(0u64, b"child".to_vec())], "the child must still be indexed at its derived count of 0, not the claimed 9" ); @@ -223,7 +226,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"psit"].as_ref(), 5, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(42i64, b"entry".to_vec())], "baseline: the entry is indexed at sum 42" ); diff --git a/grovedb/src/tests/coverage_batch_indexed_tests.rs b/grovedb/src/tests/coverage_batch_indexed_tests.rs index 8caec6bbc..d8460db26 100644 --- a/grovedb/src/tests/coverage_batch_indexed_tests.rs +++ b/grovedb/src/tests/coverage_batch_indexed_tests.rs @@ -25,6 +25,8 @@ mod tests { use grovedb_merk::{tree::AggregateData, tree_type::TreeType}; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::{BatchApplyOptions, GroveOp, QualifiedGroveDbOp, SubelementsDeletionBehavior}, tests::{common::EMPTY_PATH, make_test_grovedb, ANOTHER_TEST_LEAF, TEST_LEAF}, @@ -621,7 +623,8 @@ mod tests { assert_eq!( db.indexed_sum_top_k([TEST_LEAF, b"psit"].as_ref(), 5, true, None, grove_version) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(5i64, b"a".to_vec())], "the sum index must reflect the row inserted alongside the creation" ); @@ -657,7 +660,8 @@ mod tests { grove_version ) .unwrap() - .expect("sum top_k"), + .expect("sum top_k") + .key_pairs(), vec![(5i64, b"a".to_vec())], "the sum index must reflect the row inserted alongside the creation" ); diff --git a/grovedb/src/tests/coverage_round7_tests.rs b/grovedb/src/tests/coverage_round7_tests.rs index 508379682..fc739b12c 100644 --- a/grovedb/src/tests/coverage_round7_tests.rs +++ b/grovedb/src/tests/coverage_round7_tests.rs @@ -14,6 +14,8 @@ mod tests { use grovedb_merk::proofs::Query as MerkQuery; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, operations::proof::indexed_axis::{ @@ -1698,7 +1700,10 @@ mod tests { assert_eq!(top.len(), 2); // The partial batch must have mirrored the DERIVED count of the // new child into the secondary, ahead of the existing "a" (3). - assert_eq!(top, vec![(9, b"b".to_vec()), (3, b"a".to_vec())]); + assert_eq!( + top.key_pairs(), + vec![(9, b"b".to_vec()), (3, b"a".to_vec())] + ); assert_verify_passes(&db, grove_version); } diff --git a/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs b/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs index 8a1539bb0..290ced919 100644 --- a/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs +++ b/grovedb/src/tests/indexed_axis_nested_and_bounds_tests.rs @@ -19,6 +19,8 @@ mod tests { use grovedb_merk::proofs::query::AggregateFold; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::IndexedAxisEntry; use crate::{ @@ -147,8 +149,8 @@ mod tests { "the reconstructed root must equal the live GroveDB root" ); assert_eq!( - count_entries(&result.entries), - &vec![(1u64, b"y".to_vec()), (1u64, b"x".to_vec())], + count_entries(&result.entries).key_pairs(), + vec![(1u64, b"y".to_vec()), (1u64, b"x".to_vec())], "descending count order, ties broken by descending key" ); } @@ -354,8 +356,8 @@ mod tests { "the reconstructed root must equal the live GroveDB root" ); assert_eq!( - count_entries(&result.entries), - &vec![(1u64, b"y".to_vec()), (1u64, b"x".to_vec())], + count_entries(&result.entries).key_pairs(), + vec![(1u64, b"y".to_vec()), (1u64, b"x".to_vec())], ); } diff --git a/grovedb/src/tests/indexed_axis_offset_proof_tests.rs b/grovedb/src/tests/indexed_axis_offset_proof_tests.rs index 5d58eb3f4..79ea5e178 100644 --- a/grovedb/src/tests/indexed_axis_offset_proof_tests.rs +++ b/grovedb/src/tests/indexed_axis_offset_proof_tests.rs @@ -27,6 +27,8 @@ mod tests { use grovedb_element::indexed::IndexAxis; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::IndexedAxisEntry; use crate::{ @@ -196,7 +198,7 @@ mod tests { .expect("verify"); assert_eq!(result.skipped, 3); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[ (70i64, b"g".to_vec()), (60, b"f".to_vec()), @@ -221,7 +223,7 @@ mod tests { .expect("verify"); assert_eq!(result.skipped, 4); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[(50i64, b"e".to_vec()), (60, b"f".to_vec())] ); } @@ -255,7 +257,7 @@ mod tests { .expect("verify"); assert_eq!(result.skipped, 8); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[(20i64, b"b".to_vec()), (10, b"a".to_vec())], "the page is the walk's tail, shorter than k" ); @@ -376,7 +378,7 @@ mod tests { .expect("verify 4th biggest"); assert_eq!(result.skipped, 3); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[(70i64, b"g".to_vec())], "rank 4 descending of sums 10..100 is g(70)" ); @@ -409,7 +411,10 @@ mod tests { ) .expect("verify rank window"); assert_eq!(result.skipped, rank_zero_based as u64); - assert_eq!(entries_as_sum(&result.entries), &[(*sum, key.to_vec())]); + assert_eq!( + entries_as_sum(&result.entries).key_pairs(), + &[(*sum, key.to_vec())] + ); } } @@ -461,7 +466,7 @@ mod tests { .expect("verify mid-tie ascending"); assert_eq!(result.skipped, 4); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[ (50i64, b"t_c".to_vec()), (50, b"t_d".to_vec()), @@ -487,7 +492,7 @@ mod tests { .expect("verify mid-tie descending"); assert_eq!(result.skipped, 3); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[ (50i64, b"t_d".to_vec()), (50, b"t_c".to_vec()), @@ -541,7 +546,7 @@ mod tests { .expect("verify"); assert_eq!(result.skipped, 1); assert_eq!( - entries_as_sum(&result.entries), + entries_as_sum(&result.entries).key_pairs(), &[ (5i64, b"b".to_vec()), (5, b"c".to_vec()), diff --git a/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs b/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs index 39e18ab74..2bc17e126 100644 --- a/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs +++ b/grovedb/src/tests/indexed_axis_paginated_cost_tests.rs @@ -17,6 +17,8 @@ mod tests { use grovedb_costs::CostContext; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, operations::proof::indexed_axis::AxisEntries, @@ -686,7 +688,8 @@ mod tests { .unwrap() .expect("legacy linear read"); assert_eq!( - counted.entries, legacy, + counted.entries.key_pairs(), + legacy, "counted and legacy diverge at offset={offset} k={k} \ descending={descending}" ); @@ -839,7 +842,8 @@ mod tests { let linear_rows = linear_rows.expect("legacy linear read"); assert_eq!( - counted_page.entries, linear_rows, + counted_page.entries.key_pairs(), + linear_rows, "counted and linear paths diverged at n={n} k={k} offset={offset}" ); assert_eq!( diff --git a/grovedb/src/tests/indexed_axis_proof_tests.rs b/grovedb/src/tests/indexed_axis_proof_tests.rs index 7ed1b4cbc..65e3e804d 100644 --- a/grovedb/src/tests/indexed_axis_proof_tests.rs +++ b/grovedb/src/tests/indexed_axis_proof_tests.rs @@ -18,6 +18,8 @@ mod tests { use grovedb_merk::proofs::{query::QueryItem as MerkQueryItem, Query as MerkQuery}; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::IndexedAxisEntry; use crate::{ @@ -156,7 +158,7 @@ mod tests { .expect("verify"); let entries = entries_as_count(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (12u64, b"bob".to_vec()), (7u64, b"dave".to_vec()), @@ -184,7 +186,7 @@ mod tests { .expect("verify"); let entries = entries_as_count(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (1u64, b"carol".to_vec()), (5u64, b"alice".to_vec()), @@ -220,7 +222,10 @@ mod tests { // Descending paged after skipping 2: c(3 was top-3? no — desc top-2 = f(6), e(5); // after skip-2 of f,e → d(4), c(3). let entries = entries_as_count(&result.entries); - assert_eq!(entries, &[(4u64, b"d".to_vec()), (3u64, b"c".to_vec())]); + assert_eq!( + entries.key_pairs(), + &[(4u64, b"d".to_vec()), (3u64, b"c".to_vec())] + ); assert_eq!(result.skipped, 2); assert_eq!(result.root_hash, root_hash(&db, grove_version)); } @@ -271,7 +276,7 @@ mod tests { let entries = entries_as_count(&result.entries); // Ascending all: 1,5,10. assert_eq!( - entries, + entries.key_pairs(), &[ (1u64, b"a".to_vec()), (5u64, b"b".to_vec()), @@ -302,7 +307,7 @@ mod tests { .expect("verify"); let entries = entries_as_sum(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (10i64, b"c".to_vec()), (7, b"d".to_vec()), @@ -330,7 +335,7 @@ mod tests { .expect("verify"); let entries = entries_as_sum(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (50i64, b"d".to_vec()), (0, b"c".to_vec()), @@ -369,7 +374,10 @@ mod tests { .expect("verify"); // Descending after skip-2: 6,5 skipped → d(4), c(3) returned. let entries = entries_as_sum(&result.entries); - assert_eq!(entries, &[(4i64, b"d".to_vec()), (3, b"c".to_vec())]); + assert_eq!( + entries.key_pairs(), + &[(4i64, b"d".to_vec()), (3, b"c".to_vec())] + ); assert_eq!(result.skipped, 2); } @@ -483,7 +491,7 @@ mod tests { .expect("verify"); let entries = entries_as_sum(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (20i64, b"c".to_vec()), (10, b"b".to_vec()), @@ -518,7 +526,7 @@ mod tests { .expect("verify"); let entries = entries_as_avg(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (20i128 * SCALE, b"c".to_vec()), (10 * SCALE, b"b".to_vec()), @@ -634,7 +642,7 @@ mod tests { const SCALE: i128 = grovedb_element::indexed::AVG_FIXED_POINT_SCALE; // Descending top-3 by avg: d(30), c(20), b(10). After skip-1 → c, b. assert_eq!( - entries, + entries.key_pairs(), &[(20i128 * SCALE, b"c".to_vec()), (10 * SCALE, b"b".to_vec())] ); } @@ -886,7 +894,10 @@ mod tests { let result = GroveDb::verify_indexed_count_top_k(&proof, path, 2, true, grove_version) .expect("verify"); let entries = entries_as_count(&result.entries); - assert_eq!(entries, &[(9u64, b"b".to_vec()), (4, b"a".to_vec())]); + assert_eq!( + entries.key_pairs(), + &[(9u64, b"b".to_vec()), (4, b"a".to_vec())] + ); assert_eq!(result.root_hash, root_hash(&db, grove_version)); } @@ -1821,7 +1832,7 @@ mod tests { .expect("verify"); let entries = entries_as_count(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (10u64, b"c".to_vec()), (5u64, b"b".to_vec()), @@ -1847,7 +1858,7 @@ mod tests { .expect("verify"); let entries = entries_as_sum(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (-3i64, b"a".to_vec()), (0, b"b".to_vec()), @@ -1958,7 +1969,7 @@ mod tests { .expect("verify"); let entries = entries_as_sum(&result.entries); assert_eq!( - entries, + entries.key_pairs(), &[ (15i64, b"c".to_vec()), (10, b"b".to_vec()), @@ -2503,7 +2514,10 @@ mod tests { .expect("prove"); let result = GroveDb::verify_indexed_count_top_k(&proof, path, 1, true, grove_version) .expect("verify"); - assert_eq!(entries_as_count(&result.entries), &[(99u64, b"b".to_vec())]); + assert_eq!( + entries_as_count(&result.entries).key_pairs(), + &[(99u64, b"b".to_vec())] + ); } // ---------- top_k proof-bytes / path rejections ---------- @@ -2688,7 +2702,7 @@ mod tests { GroveDb::verify_indexed_count_top_k_paginated(&proof, path, 2, 1, false, grove_version) .expect("verify"); assert_eq!( - entries_as_count(&result.entries), + entries_as_count(&result.entries).key_pairs(), &[(2u64, b"b".to_vec()), (3u64, b"c".to_vec())] ); assert_eq!(result.skipped, 1); @@ -2821,7 +2835,7 @@ mod tests { .expect("verify"); // Descending: d(4), c(3), b(2), a(1). Skip 1 (d), take 2: c, b. assert_eq!( - entries_as_count(&result.entries), + entries_as_count(&result.entries).key_pairs(), &[(3u64, b"c".to_vec()), (2u64, b"b".to_vec())] ); assert_eq!(result.skipped, 1); @@ -3126,7 +3140,8 @@ mod tests { .expect("verify"); let got = entries_as_count(&result.entries); assert_eq!(got.len(), 1); - assert_eq!(got[0], (5u64, b"alice".to_vec())); + assert_eq!(got[0].ordering_value, 5u64); + assert_eq!(got[0].primary_key, b"alice".to_vec()); } #[test] diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index 3683e8be1..a9e6d60f5 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -804,6 +804,170 @@ mod tests { assert_eq!(entries[0].primary_key, b"ref".to_vec()); } + /// A MULTI-HOP reference chain out of an indexed primary. + /// + /// GroveDB references commit the TERMINAL's value hash, not the next + /// hop's — `follow_reference_get_value_hash` recurses past every + /// intermediate reference before the hash is baked into + /// `PutCombinedReference`. A chain fold that composes hop-by-hop + /// happens to agree at one hop and diverges at two, so one-hop + /// coverage cannot catch it. + #[test] + fn a_multi_hop_reference_primary_resolves_and_verifies() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"terminal", + Element::new_item(b"terminal-value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("terminal"); + db.insert( + [TEST_LEAF].as_ref(), + b"middle", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"terminal".to_vec(), + ])), + None, + None, + grove_version, + ) + .unwrap() + .expect("middle hop"); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"alias", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"middle".to_vec(), + ])), + None, + grove_version, + ) + .unwrap() + .expect("two-hop primary"); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!(issues.is_empty(), "multi-hop chain reported: {issues:?}"); + + let expected = Element::new_item(b"terminal-value".to_vec()); + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + + let direct = db + .indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("direct top_k"); + assert_eq!(direct.len(), 1); + assert_eq!(direct[0].value, expected, "direct read follows both hops"); + + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("prove a two-hop primary"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("a two-hop reference chain must verify"); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + assert_eq!(entries.len(), 1); + assert_eq!( + entries[0].value, expected, + "the proved read must reach the same terminal as the direct read" + ); + } + + /// A RELATIVE (sibling) reference out of an indexed primary. + /// + /// `SiblingReference` resolves by appending its key to the CURRENT + /// PATH, so the path handed to resolution must be the node's parent, + /// not the node's own qualified path. Passing one segment too deep + /// sends resolution looking for a child underneath the entry itself. + /// An `UpstreamRootHeightReference` masks the error — it truncates to + /// the first N segments and lands in the same place either way — so + /// this needs its own case. + #[test] + fn a_sibling_reference_primary_resolves_and_verifies() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"target", + Element::new_item(b"sibling-value".to_vec()), + None, + grove_version, + ) + .unwrap() + .expect("sibling target"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"alias", + Element::new_reference(ReferencePathType::SiblingReference(b"target".to_vec())), + None, + grove_version, + ) + .unwrap() + .expect("sibling-reference primary"); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!(issues.is_empty(), "sibling chain reported: {issues:?}"); + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let expected = Element::new_item(b"sibling-value".to_vec()); + + let direct = db + .indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("direct top_k"); + let alias = direct + .iter() + .find(|e| e.primary_key == b"alias".to_vec()) + .expect("alias row"); + assert_eq!(alias.value, expected, "direct read resolves the sibling"); + + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("prove a sibling-reference primary"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("a sibling reference chain must verify"); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + let alias = entries + .iter() + .find(|e| e.primary_key == b"alias".to_vec()) + .expect("alias row in proof"); + assert_eq!( + alias.value, expected, + "the proved read resolves the sibling" + ); + } + /// Ordinary user references keep their own semantics. The one-hop /// immediate-node rule is dedicated indexed-tree behaviour, so it must /// not leak into how a normal reference elsewhere in the grove is diff --git a/grovedb/src/tests/indexed_tree_secondary_drift_tests.rs b/grovedb/src/tests/indexed_tree_secondary_drift_tests.rs index b0e360543..07c03bb73 100644 --- a/grovedb/src/tests/indexed_tree_secondary_drift_tests.rs +++ b/grovedb/src/tests/indexed_tree_secondary_drift_tests.rs @@ -28,6 +28,8 @@ mod tests { use grovedb_storage::{Storage, StorageBatch}; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ tests::{common::EMPTY_PATH, make_test_grovedb, TempGroveDb, TEST_LEAF}, Element, Error, @@ -289,7 +291,7 @@ mod tests { .unwrap() .expect("top_k"); assert_eq!( - pristine_listing, + pristine_listing.key_pairs(), vec![(1u64, b"a".to_vec()), (1u64, b"b".to_vec())], "baseline listing" ); @@ -374,7 +376,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, false, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"b".to_vec())], "the drifted index must be missing 'a'" ); @@ -388,7 +391,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, false, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"a".to_vec()), (1u64, b"b".to_vec())], "'a' must be back in the count index at count 1" ); @@ -430,7 +434,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(7u64, b"a".to_vec()), (1u64, b"b".to_vec())], "the drifted index must rank 'a' at 7" ); @@ -444,7 +449,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, false, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"a".to_vec()), (1u64, b"b".to_vec())], "'a' must be back at count 1" ); @@ -574,7 +580,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(3u64, b"big".to_vec()), (1u64, b"small".to_vec())], "baseline: the children rank by their own descendant counts" ); @@ -590,7 +597,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"small".to_vec())], "the drifted index must be missing 'big'" ); @@ -602,7 +610,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, true, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(3u64, b"big".to_vec()), (1u64, b"small".to_vec())], "reconcile must index each child at its own aggregate count, not at 1" ); @@ -773,7 +782,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 10, false, None, gv) .unwrap() - .expect("top_k"), + .expect("top_k") + .key_pairs(), vec![(1u64, b"z".to_vec())], "only 'z' must be indexed" ); @@ -850,7 +860,8 @@ mod tests { db.indexed_count_top_k_paginated([TEST_LEAF, b"pcit"].as_ref(), 1, 1, true, None, gv) .unwrap() .expect("counted skip passes the malformed row without decoding it") - .entries, + .entries + .key_pairs(), vec![(1u64, b"b".to_vec())], "descending order is [0xff…, b, a]; the malformed row is counted at offset 0", ); @@ -886,7 +897,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"pcit"].as_ref(), 2, false, None, gv) .unwrap() - .expect("the first two rows are well formed"), + .expect("the first two rows are well formed") + .key_pairs(), vec![(1u64, b"a".to_vec()), (1u64, b"b".to_vec())], ); } @@ -1328,7 +1340,8 @@ mod tests { db.indexed_count_top_k_paginated([TEST_LEAF, b"pcit"].as_ref(), 2, 0, false, None, gv) .unwrap() .expect("offset-0 path reads physical rows") - .entries, + .entries + .key_pairs(), vec![(1u64, b"a".to_vec()), (1u64, b"b".to_vec())], ); } diff --git a/grovedb/src/tests/indexed_tree_security_regression_tests.rs b/grovedb/src/tests/indexed_tree_security_regression_tests.rs index ff6aec47a..34fd0f425 100644 --- a/grovedb/src/tests/indexed_tree_security_regression_tests.rs +++ b/grovedb/src/tests/indexed_tree_security_regression_tests.rs @@ -5,6 +5,8 @@ use grovedb_merk::tree_type::TreeType; use grovedb_version::version::GroveVersion; use tempfile::TempDir; +use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::{QualifiedGroveDbOp, SubelementsDeletionBehavior}, tests::{common::EMPTY_PATH, make_test_grovedb, TEST_LEAF}, @@ -108,7 +110,8 @@ fn psit_rejects_big_sum_tree_at_i64_boundary() { assert_eq!( db.indexed_sum_top_k([b"psit".as_slice()].as_ref(), 10, true, None, grove_version,) .unwrap() - .unwrap(), + .unwrap() + .key_pairs(), vec![(42, b"sum".to_vec())] ); } @@ -154,7 +157,8 @@ fn delete_tree_rejects_declared_type_mismatch() { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 10, true, None, grove_version,) .unwrap() - .unwrap(), + .unwrap() + .key_pairs(), vec![(1, b"row".to_vec())] ); assert_verify_passes(&db, grove_version); @@ -418,7 +422,8 @@ fn batch_count_changes_remove_all_old_secondary_rows_first() { grove_version, ) .unwrap() - .unwrap(), + .unwrap() + .key_pairs(), vec![(2, b"a".to_vec()), (2, b"b".to_vec())] ); assert_verify_passes(&db, grove_version); @@ -575,7 +580,7 @@ fn batch_rejects_rootless_aggregate_child_under_indexed_primary() { .unwrap() .expect("top_k"); assert_eq!( - top, + top.key_pairs(), vec![(9, b"b".to_vec())], "the derived count must reach the secondary index" ); diff --git a/grovedb/src/tests/provable_count_indexed_tree_tests.rs b/grovedb/src/tests/provable_count_indexed_tree_tests.rs index ff97d1c81..be6b40d4b 100644 --- a/grovedb/src/tests/provable_count_indexed_tree_tests.rs +++ b/grovedb/src/tests/provable_count_indexed_tree_tests.rs @@ -21,6 +21,8 @@ mod tests { use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, tests::{make_test_grovedb, TEST_LEAF}, @@ -824,7 +826,8 @@ mod tests { assert_eq!( db.indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) .unwrap() - .expect("count top_k"), + .expect("count top_k") + .key_pairs(), vec![(1u64, b"row".to_vec())], "the count index must reflect the row inserted alongside the creation \ (a plain Item contributes count 1; only an empty tree indexes at 0)" @@ -975,7 +978,7 @@ mod tests { .unwrap() .expect("top-k descending"); assert_eq!( - top3, + top3.key_pairs(), vec![ (20u64, b"eve".to_vec()), (12u64, b"bob".to_vec()), @@ -989,7 +992,7 @@ mod tests { .unwrap() .expect("top-k ascending"); assert_eq!( - bottom2, + bottom2.key_pairs(), vec![(1u64, b"carol".to_vec()), (5u64, b"alice".to_vec())] ); } @@ -1040,7 +1043,7 @@ mod tests { .unwrap() .expect("page 1"); assert_eq!( - page1.entries, + page1.entries.key_pairs(), vec![(20u64, b"eve".to_vec()), (12u64, b"bob".to_vec())] ); @@ -1057,7 +1060,7 @@ mod tests { .unwrap() .expect("page 2"); assert_eq!( - page2.entries, + page2.entries.key_pairs(), vec![(7u64, b"dave".to_vec()), (5u64, b"alice".to_vec())] ); @@ -1073,7 +1076,7 @@ mod tests { ) .unwrap() .expect("page 3"); - assert_eq!(page3.entries, vec![(1u64, b"carol".to_vec())]); + assert_eq!(page3.entries.key_pairs(), vec![(1u64, b"carol".to_vec())]); // Offset beyond total → empty. let beyond = db @@ -1129,7 +1132,7 @@ mod tests { .unwrap() .expect("range"); assert_eq!( - in_range, + in_range.key_pairs(), vec![ (5u64, b"alice".to_vec()), (7u64, b"dave".to_vec()), @@ -1151,7 +1154,7 @@ mod tests { .unwrap() .expect("range desc"); assert_eq!( - in_range_desc, + in_range_desc.key_pairs(), vec![ (12u64, b"bob".to_vec()), (7u64, b"dave".to_vec()), @@ -1172,7 +1175,7 @@ mod tests { ) .unwrap() .expect("exact"); - assert_eq!(exact, vec![(12u64, b"bob".to_vec())]); + assert_eq!(exact.key_pairs(), vec![(12u64, b"bob".to_vec())]); // lo > hi: empty. let empty = db diff --git a/grovedb/src/tests/provable_count_provable_sum_indexed_tree_tests.rs b/grovedb/src/tests/provable_count_provable_sum_indexed_tree_tests.rs index 864134821..3a99b48e6 100644 --- a/grovedb/src/tests/provable_count_provable_sum_indexed_tree_tests.rs +++ b/grovedb/src/tests/provable_count_provable_sum_indexed_tree_tests.rs @@ -19,6 +19,8 @@ mod tests { use grovedb_element::indexed::IndexAxis; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, tests::{make_test_grovedb, TEST_LEAF}, @@ -1070,7 +1072,7 @@ mod tests { .unwrap() .expect("top-k count"); assert_eq!( - top3, + top3.key_pairs(), vec![ (5u64, b"carol".to_vec()), (4, b"bob".to_vec()), @@ -1100,7 +1102,7 @@ mod tests { .unwrap() .expect("range"); assert_eq!( - in_range, + in_range.key_pairs(), vec![ (2u64, b"alice".to_vec()), (3, b"eve".to_vec()), @@ -1157,7 +1159,7 @@ mod tests { .unwrap() .expect("top-k sum"); assert_eq!( - top3, + top3.key_pairs(), vec![ (100i64, b"bob".to_vec()), (10, b"alice".to_vec()), @@ -1177,7 +1179,7 @@ mod tests { .unwrap() .expect("asc"); assert_eq!( - asc3, + asc3.key_pairs(), vec![ (-25i64, b"carol".to_vec()), (0, b"dave".to_vec()), @@ -1207,7 +1209,7 @@ mod tests { .unwrap() .expect("sum range"); assert_eq!( - in_range, + in_range.key_pairs(), vec![ (0i64, b"dave".to_vec()), (9, b"eve".to_vec()), @@ -1269,7 +1271,7 @@ mod tests { .unwrap() .expect("top-k avg"); assert_eq!( - top3, + top3.key_pairs(), vec![ (25 * AVG_SCALE, b"bob".to_vec()), (5 * AVG_SCALE, b"alice".to_vec()), @@ -1289,7 +1291,7 @@ mod tests { .unwrap() .expect("asc avg"); assert_eq!( - asc3, + asc3.key_pairs(), vec![ (-5 * AVG_SCALE, b"carol".to_vec()), (0, b"dave".to_vec()), @@ -1319,7 +1321,7 @@ mod tests { .unwrap() .expect("avg range"); assert_eq!( - in_range, + in_range.key_pairs(), vec![ (0i128, b"dave".to_vec()), (3 * AVG_SCALE, b"eve".to_vec()), @@ -1340,7 +1342,7 @@ mod tests { ) .unwrap() .expect("exact"); - assert_eq!(exact, vec![(3 * AVG_SCALE, b"eve".to_vec())]); + assert_eq!(exact.key_pairs(), vec![(3 * AVG_SCALE, b"eve".to_vec())]); // lo > hi: empty. let empty = db @@ -1395,7 +1397,7 @@ mod tests { .unwrap() .expect("page 1"); assert_eq!( - page1.entries, + page1.entries.key_pairs(), vec![ (25 * AVG_SCALE, b"bob".to_vec()), (5 * AVG_SCALE, b"alice".to_vec()), @@ -1415,7 +1417,7 @@ mod tests { .unwrap() .expect("page 2"); assert_eq!( - page2.entries, + page2.entries.key_pairs(), vec![(3 * AVG_SCALE, b"eve".to_vec()), (0, b"dave".to_vec())] ); @@ -1461,7 +1463,7 @@ mod tests { ) .unwrap() .expect("zero only"); - assert_eq!(zero_only, vec![(0i128, b"dave".to_vec())]); + assert_eq!(zero_only.key_pairs(), vec![(0i128, b"dave".to_vec())]); } #[test] @@ -1490,7 +1492,7 @@ mod tests { .expect("asc"); // Both share avg = 7*SCALE; tie-break by original_key ascending. assert_eq!( - asc, + asc.key_pairs(), vec![ (7 * AVG_SCALE, b"aaa".to_vec()), (7 * AVG_SCALE, b"zzz".to_vec()), @@ -1609,7 +1611,7 @@ mod tests { ) .unwrap() .expect("count"); - assert_eq!(by_count, vec![(1u64, b"row".to_vec())]); + assert_eq!(by_count.key_pairs(), vec![(1u64, b"row".to_vec())]); let by_sum = db .indexed_sum_top_k( [TEST_LEAF, b"pcpsit"].as_ref(), @@ -1620,7 +1622,7 @@ mod tests { ) .unwrap() .expect("sum"); - assert_eq!(by_sum, vec![(17i64, b"row".to_vec())]); + assert_eq!(by_sum.key_pairs(), vec![(17i64, b"row".to_vec())]); let by_avg = db .indexed_avg_top_k( [TEST_LEAF, b"pcpsit"].as_ref(), @@ -1632,7 +1634,7 @@ mod tests { .unwrap() .expect("avg"); // avg = floor(17 * SCALE / 1) = 17 * SCALE. - assert_eq!(by_avg, vec![(17 * AVG_SCALE, b"row".to_vec())]); + assert_eq!(by_avg.key_pairs(), vec![(17 * AVG_SCALE, b"row".to_vec())]); } // ----------------------------------------------------------------- diff --git a/grovedb/src/tests/provable_sum_indexed_tree_tests.rs b/grovedb/src/tests/provable_sum_indexed_tree_tests.rs index 324382ab6..3d855d029 100644 --- a/grovedb/src/tests/provable_sum_indexed_tree_tests.rs +++ b/grovedb/src/tests/provable_sum_indexed_tree_tests.rs @@ -18,6 +18,8 @@ mod tests { use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ batch::QualifiedGroveDbOp, tests::{make_test_grovedb, TEST_LEAF}, @@ -825,7 +827,7 @@ mod tests { .unwrap() .expect("top-k desc"); assert_eq!( - top3, + top3.key_pairs(), vec![ (100i64, b"frank".to_vec()), (12, b"bob".to_vec()), @@ -839,7 +841,7 @@ mod tests { .unwrap() .expect("bottom-2"); assert_eq!( - bottom2, + bottom2.key_pairs(), vec![(-7i64, b"carol".to_vec()), (-1, b"eve".to_vec())] ); } @@ -888,7 +890,7 @@ mod tests { .unwrap() .expect("asc"); assert_eq!( - asc, + asc.key_pairs(), vec![ (-42i64, b"neg".to_vec()), (0, b"zero".to_vec()), @@ -918,7 +920,7 @@ mod tests { .unwrap() .expect("page 1"); assert_eq!( - page1.entries, + page1.entries.key_pairs(), vec![(100i64, b"frank".to_vec()), (12, b"bob".to_vec())] ); @@ -934,7 +936,7 @@ mod tests { .unwrap() .expect("page 2"); assert_eq!( - page2.entries, + page2.entries.key_pairs(), vec![(5i64, b"alice".to_vec()), (0, b"dave".to_vec())] ); @@ -992,7 +994,7 @@ mod tests { .unwrap() .expect("range"); assert_eq!( - in_range, + in_range.key_pairs(), vec![ (-1i64, b"eve".to_vec()), (0, b"dave".to_vec()), @@ -1015,7 +1017,7 @@ mod tests { .unwrap() .expect("desc"); assert_eq!( - desc, + desc.key_pairs(), vec![ (12i64, b"bob".to_vec()), (5, b"alice".to_vec()), @@ -1037,7 +1039,7 @@ mod tests { ) .unwrap() .expect("exact"); - assert_eq!(exact, vec![(12i64, b"bob".to_vec())]); + assert_eq!(exact.key_pairs(), vec![(12i64, b"bob".to_vec())]); // lo > hi: empty. let empty = db @@ -1105,7 +1107,10 @@ mod tests { ) .unwrap() .expect("neg range"); - assert_eq!(neg, vec![(-7i64, b"carol".to_vec()), (-1, b"eve".to_vec())]); + assert_eq!( + neg.key_pairs(), + vec![(-7i64, b"carol".to_vec()), (-1, b"eve".to_vec())] + ); } #[test] diff --git a/grovedb/src/tests/verify_grovedb_indexed_tests.rs b/grovedb/src/tests/verify_grovedb_indexed_tests.rs index e7e0a936b..e0692cc29 100644 --- a/grovedb/src/tests/verify_grovedb_indexed_tests.rs +++ b/grovedb/src/tests/verify_grovedb_indexed_tests.rs @@ -16,6 +16,8 @@ mod tests { use grovedb_storage::{Storage, StorageBatch}; use grovedb_version::version::GroveVersion; + use crate::IndexedAxisEntrySliceExt; + use crate::{ operations::insert::InsertOptions, tests::{make_test_grovedb, TEST_LEAF}, @@ -1782,7 +1784,7 @@ mod tests { .unwrap() .expect("top-k over distinct derived counts"); assert_eq!( - top, + top.key_pairs(), (0..15u64) .rev() .map(|i| (i, format!("k{:02}", i).into_bytes())) @@ -1850,7 +1852,7 @@ mod tests { .unwrap() .expect("ascending top-k over tied derived counts"); assert_eq!( - asc, + asc.key_pairs(), (0..10) .map(|i| (SHARED_COUNT, format!("k{:02}", i).into_bytes())) .collect::>() diff --git a/merk/src/proofs/query/count_offset/tests.rs b/merk/src/proofs/query/count_offset/tests.rs index a8eaf8e60..bc7f9a37a 100644 --- a/merk/src/proofs/query/count_offset/tests.rs +++ b/merk/src/proofs/query/count_offset/tests.rs @@ -185,8 +185,6 @@ fn returned_items_carry_full_committed_payload() { value: vec![5u8], value_hash: crate::tree::value_hash(&[5u8]).unwrap(), child_hash_verified: false, - // A directly-valued row, not a resolved reference. - reference_element_hash: None, }; assert_eq!( verified.returned_items[0], expected_f, diff --git a/merk/src/proofs/query/count_offset/verify.rs b/merk/src/proofs/query/count_offset/verify.rs index f0b42ad17..1bfa31d30 100644 --- a/merk/src/proofs/query/count_offset/verify.rs +++ b/merk/src/proofs/query/count_offset/verify.rs @@ -106,19 +106,6 @@ pub struct CountOffsetReturnedItem { /// `child_hash_verified = true` for non-empty trees); callers must /// not silently treat a `false` here as `true`. pub child_hash_verified: bool, - /// For a resolved-reference row (`KVRefValueHash*`), the value hash - /// of the REFERENCE element itself — the first half of the row's - /// committed `combine_hash(reference_element_hash, H(value))`. - /// `None` for a directly-valued row. - /// - /// This is what lets a caller authenticate the reference's own - /// content without the proof carrying its bytes: a caller that knows - /// the canonical shape a row must have can reconstruct those bytes - /// and check `value_hash(reconstructed) == reference_element_hash`. - /// GroveDB's indexed-axis verifier does exactly that, which is how a - /// row's `SiblingReference(primary_key)` / hop budget / carried sum - /// get authenticated rather than assumed. - pub reference_element_hash: Option, } /// The verifier's reconstructed view of an offset-paginated count-tree @@ -611,7 +598,6 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: vh, - reference_element_hash: None, }) } Node::KVCountSum(key, value, _, _) => { @@ -637,7 +623,6 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: vh, - reference_element_hash: None, }) } Node::KVValueHashFeatureType(key, value, vh, _) => { @@ -711,7 +696,6 @@ fn classify_self<'a>( key: key.as_slice(), value: value.as_slice(), value_hash: *vh, - reference_element_hash: None, }) } // ─── Resolved-reference returns ────────────────────────────── @@ -755,7 +739,6 @@ fn classify_self<'a>( &compute_value_hash(value.as_slice()).unwrap(), ) .unwrap(), - reference_element_hash: Some(*reference_element_hash), }) } Node::KVRefValueHashCountSum(key, value, reference_element_hash, _, _) => { @@ -779,7 +762,6 @@ fn classify_self<'a>( &compute_value_hash(value.as_slice()).unwrap(), ) .unwrap(), - reference_element_hash: Some(*reference_element_hash), }) } Node::KVValueHash(key, value, _) => { @@ -841,10 +823,6 @@ enum BoundaryKind<'a> { /// proof-carried value_hash (tree-flavored entries store /// `combine_hash(H(value), child_root)`). value_hash: CryptoHash, - /// For a resolved-reference return, the reference element's own - /// value hash; `None` for a directly-valued entry. See - /// [`CountOffsetReturnedItem::reference_element_hash`]. - reference_element_hash: Option, }, } @@ -886,7 +864,6 @@ fn apply_self_state(disposition: &BoundaryKind<'_>, state: &mut VerifyState) -> key, value, value_hash, - reference_element_hash, } => { if state.offset_remaining > 0 { return Err(Error::InvalidProofError( @@ -919,7 +896,6 @@ fn apply_self_state(disposition: &BoundaryKind<'_>, state: &mut VerifyState) -> // right behavior given that we don't carry a child // hash to validate. child_hash_verified: false, - reference_element_hash: *reference_element_hash, }); Ok(()) } From a24a9704139c8e73dc549bf7fe09352846efe684 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 06:19:10 +0700 Subject: [PATCH 06/18] refactor: refresh indexed rows inside the propagation walk (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts #816's factoring — the per-entry row refresh moves into the propagation loop, so a typed write path opts in with one call instead of ~20 lines of deferred-seed plumbing. Net −79 lines while adding a call site. The fiddly part was never the mirror; it was the deferred per-axis root state. Single-axis variants seed one slot and PCPSIT another, and seeding the wrong one leaves state set for an iteration with no indexed element to apply it to. That belongs in the one place already managing it. **This caught a fifth write path I had missed.** `replace_subtree_root` rewrites an entry in place and then propagates, exactly like the four non-Merk appends, so it left the canonical row bound to a commitment that no longer existed. #816 covers it; I did not. That is the factoring argument demonstrated rather than asserted: with the refresh inside the walk a new caller is one line and cannot forget, whereas per-call-site plumbing makes every new site opt-in and missable — which is how I missed this one. **Kept the old-state capture rather than refreshing in place.** #816's in-loop refresh passes the same aggregates on both sides, which only rewrites the row at its existing key. That is sound for the non-Merk appends, whose aggregates provably cannot change, but not for `replace_subtree_root`: its element is CALLER-SUPPLIED, so its aggregates — and therefore the row's sort key — can differ from what was there, and an in-place refresh would strand the old row at the old key. Callers capture pre-rewrite state with `capture_indexed_entry_state` (one line) and the walk applies a full old → new transition. The new test states a count the subtree's contents do not support, which moves the sort key. It asserts the row MOVED and that no indexed-row sentinel appears — while deliberately tolerating the child's own aggregate mismatch, which is the hash-vs-state correctness this unsafe API hands to the caller. Reverting the fix makes it fail with the row stranded at the old count, so it tests what it claims to. Default suite: 43 targets, 2739 grovedb tests. With `unsafe-dump-load`: 2742. `clippy -D warnings` passes on both feature sets. Per-row proof cost unchanged at 83 bytes. --- grovedb/src/lib.rs | 427 ++++++++---------- grovedb/src/operations/bulk_append_tree.rs | 22 +- grovedb/src/operations/commitment_tree.rs | 22 +- grovedb/src/operations/dense_tree.rs | 22 +- grovedb/src/operations/mmr_tree.rs | 23 +- .../src/operations/replace_subtree_root.rs | 23 +- .../src/tests/indexed_reference_row_tests.rs | 136 ++++++ 7 files changed, 366 insertions(+), 309 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index e8dca4fbc..bb85bda02 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -738,143 +738,6 @@ impl GroveDb { ) } - /// Mirror ONE entry of an indexed primary into every axis it - /// configures, returning each axis's post-mirror `(tag, root_hash, - /// root_key)` so the caller can hand it to propagation as the initial - /// deferred state. - /// - /// This exists for the direct non-Merk append APIs (MMR, commitment, - /// bulk-append, dense). Those write the updated element straight into - /// the primary Merk and only then start propagating, so the - /// propagation walk — which mirrors entries it discovers as it climbs - /// — never sees the entry that actually moved. That was harmless while - /// rows were aggregate-only (a non-Merk child's count is a constant - /// `1`, so its row never moved) and is not harmless now: a canonical - /// row binds the primary node's commitment, and an append rewrites - /// exactly that. - /// - /// `old_state` must be captured BEFORE the entry was rewritten. - pub(crate) fn mirror_indexed_primary_entry<'b, B: AsRef<[u8]>>( - &self, - primary_path: SubtreePath<'b, B>, - primary_merk: &Merk, - indexed_element: &Element, - item_key: &[u8], - old_state: Option, - transaction: &Transaction, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult>)>, Error> { - use crate::operations::indexed_tree::{ - mirror_indexed_axis_to_secondary, IndexedEntryState, - }; - - let mut cost = OperationCost::default(); - - let axes: Vec<(u8, Option>)> = match indexed_element.underlying() { - Element::ProvableCountIndexedTree(_, secondary, ..) => { - vec![( - grovedb_element::indexed::IndexAxis::Count.tag(), - secondary.clone(), - )] - } - Element::ProvableSumIndexedTree(_, secondary, ..) => { - vec![( - grovedb_element::indexed::IndexAxis::Sum.tag(), - secondary.clone(), - )] - } - Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => axes.clone(), - _ => { - return Err(Error::CorruptedCodeExecution( - "mirror_indexed_primary_entry called with a non-indexed element", - )) - .wrap_with_cost(cost); - } - }; - - // Post-write state of the entry, read from the primary Merk the - // caller already updated. - let new_state = { - let maybe_element = cost_return_on_error!( - &mut cost, - Element::get_optional(primary_merk, item_key, true, grove_version) - .map_err(Error::MerkError) - ); - match maybe_element { - None => None, - Some(element) => { - let value_hash = cost_return_on_error!( - &mut cost, - primary_merk - .get_value_hash( - item_key, - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(Error::MerkError) - ); - let (count, sum) = element.count_sum_value_or_default(); - value_hash.map(|value_hash| IndexedEntryState { - count, - sum, - value_hash, - }) - } - } - }; - - let mut staged: Vec<(u8, Hash, Option>)> = Vec::with_capacity(axes.len()); - for (tag, secondary_root_key) in axes { - let axis = cost_return_on_error_no_add!( - cost, - grovedb_element::indexed::IndexAxis::try_from_tag(tag).map_err(|e| { - Error::CorruptedData(format!( - "invalid axis tag on an indexed element during entry mirror: {e}" - )) - }) - ); - let mut secondary_merk = cost_return_on_error!( - &mut cost, - self.open_indexed_secondary_at_path( - primary_path.clone(), - axis, - secondary_root_key, - transaction, - Some(batch), - grove_version, - ) - ); - cost_return_on_error!( - &mut cost, - mirror_indexed_axis_to_secondary( - &mut secondary_merk, - axis, - item_key, - old_state, - new_state, - grove_version, - ) - ); - let (sec_hash, sec_key, _) = cost_return_on_error!( - &mut cost, - secondary_merk - .root_hash_key_and_aggregate_data() - .map_err(Error::MerkError) - ); - staged.push((tag, sec_hash, sec_key)); - } - - Ok(staged).wrap_with_cost(cost) - } - - /// Snapshot an entry's indexed state before a non-Merk append - /// rewrites it, returning `None` when the parent is not an indexed - /// primary (in which case there is nothing to mirror). - /// - /// Pairs with [`Self::mirror_indexed_entry_and_seed`]; see its docs - /// for why the direct non-Merk append APIs need this at all. pub(crate) fn capture_indexed_entry_state<'db>( primary_merk: &Merk>, key: &[u8], @@ -907,95 +770,52 @@ impl GroveDb { .wrap_with_cost(cost) } - /// Mirror one rewritten entry of an indexed primary into its axis - /// secondaries and produce the seeds propagation needs. + /// Propagate after an in-place rewrite of ONE entry under an indexed + /// primary, refreshing that entry's canonical secondary row on the way + /// up. /// - /// The direct non-Merk append APIs (MMR, commitment, bulk-append, - /// dense) write the updated element straight into the primary Merk - /// and only then start propagating, so the propagation walk — which - /// mirrors entries it discovers as it climbs — never sees the entry - /// that actually moved. That was harmless while rows were - /// aggregate-only (a non-Merk child's count is a constant `1`, so its - /// row never moved) and is not harmless now: a canonical row binds the - /// primary node's commitment, and an append rewrites exactly that. + /// The typed direct write paths — the four non-Merk appends and + /// `replace_subtree_root` — write their updated element straight into + /// the primary Merk and only then propagate. The propagation walk + /// mirrors entries it *discovers* as it climbs, so it never sees the + /// entry that actually moved. That was harmless while rows were + /// aggregate-only (these mutations leave `(count, sum)` alone) and is + /// not harmless now: a canonical row binds the primary node's + /// commitment, and every one of these rewrites moves exactly that. /// - /// Returns `(initial_deferred_secondary, initial_deferred_axes)`. The - /// two are not interchangeable — propagation rebuilds a single-axis - /// element from the former and a PCPSIT from the latter, and seeding - /// the wrong one leaves the other set for a later iteration that has - /// no indexed element to apply it to. - #[allow(clippy::type_complexity)] - pub(crate) fn mirror_indexed_entry_and_seed<'b, 'db, B: AsRef<[u8]>>( + /// Doing the refresh INSIDE the walk rather than at each call site is + /// deliberate. The fiddly part is the deferred per-axis root state — + /// single-axis variants seed one slot, PCPSIT seeds another, and + /// getting it wrong leaves state set for an iteration with no indexed + /// element to apply it to. That belongs in the one place that already + /// manages it, so a new caller is a one-line change and cannot forget. + /// + /// `old_state` must be captured with + /// [`Self::capture_indexed_entry_state`] BEFORE the rewrite. It is what + /// lets this handle a rewrite that MOVES the row: `replace_subtree_root` + /// takes a caller-supplied element, so its aggregates — and therefore + /// the row's sort key — can differ from what was there. Refreshing in + /// place without it would leave the old row stranded at the old key. + pub(crate) fn propagate_changes_with_transaction_refreshing_indexed_row<'b, B: AsRef<[u8]>>( &self, - primary_path: SubtreePath<'b, B>, - primary_merk: &Merk>, - key: &[u8], + merk_cache: HashMap, Merk>, + path: SubtreePath<'b, B>, + changed_key: &[u8], old_state: Option, transaction: &Transaction, batch: &StorageBatch, grove_version: &GroveVersion, - ) -> CostResult< - ( - Option<(Hash, Option>)>, - Option>)>>, - ), - Error, - > { - let mut cost = OperationCost::default(); - if !primary_merk.tree_type.is_indexed_primary() { - return Ok((None, None)).wrap_with_cost(cost); - } - - let (grandparent_path, indexed_key) = cost_return_on_error_no_add!( - cost, - primary_path - .derive_parent() - .ok_or(Error::CorruptedCodeExecution( - "an indexed primary requires a grandparent holding its indexed element", - )) - ); - let indexed_element = { - let grandparent_merk = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - grandparent_path, - transaction, - Some(batch), - grove_version, - ) - ); - cost_return_on_error!( - &mut cost, - Element::get(&grandparent_merk, indexed_key, true, grove_version) - .map_err(Error::MerkError) - ) - }; - let is_multi_axis = matches!( - indexed_element.underlying(), - Element::ProvableCountProvableSumIndexedTree(..) - ); - let staged = cost_return_on_error!( - &mut cost, - self.mirror_indexed_primary_entry( - primary_path, - primary_merk, - &indexed_element, - key, - old_state, - transaction, - batch, - grove_version, - ) - ); - let seeds = if is_multi_axis { - (None, Some(staged)) - } else { - ( - staged.first().map(|(_, hash, key)| (*hash, key.clone())), - None, - ) - }; - Ok(seeds).wrap_with_cost(cost) + ) -> CostResult<(), Error> { + self.propagate_changes_inner( + merk_cache, + path, + None, + None, + Some((changed_key, old_state)), + transaction, + batch, + grove_version, + ) } pub(crate) fn propagate_changes_with_transaction_with_initial_deferred<'b, B: AsRef<[u8]>>( @@ -1007,33 +827,33 @@ impl GroveDb { batch: &StorageBatch, grove_version: &GroveVersion, ) -> CostResult<(), Error> { - self.propagate_changes_with_transaction_with_initial_deferred_axes( + self.propagate_changes_inner( merk_cache, path, initial_deferred_secondary, None, + None, transaction, batch, grove_version, ) } - /// As [`Self::propagate_changes_with_transaction_with_initial_deferred`], - /// but also seeds the MULTI-AXIS deferred state a PCPSIT primary needs. - /// A single-axis seed cannot express a PCPSIT's per-axis roots, so the - /// direct non-Merk append APIs — which mirror their entry themselves - /// via [`Self::mirror_indexed_primary_entry`] before propagating — go - /// through here. + /// The one propagation walk. `changed_indexed_entry` refreshes a single + /// rewritten entry's canonical row when the walk reaches the indexed + /// primary holding it — see + /// [`Self::propagate_changes_with_transaction_refreshing_indexed_row`]. #[allow(clippy::too_many_arguments)] - pub(crate) fn propagate_changes_with_transaction_with_initial_deferred_axes< - 'b, - B: AsRef<[u8]>, - >( + fn propagate_changes_inner<'b, B: AsRef<[u8]>>( &self, mut merk_cache: HashMap, Merk>, path: SubtreePath<'b, B>, initial_deferred_secondary: Option<(Hash, Option>)>, initial_deferred_axes: Option>)>>, + changed_indexed_entry: Option<( + &[u8], + Option, + )>, transaction: &Transaction, batch: &StorageBatch, grove_version: &GroveVersion, @@ -1090,6 +910,153 @@ impl GroveDb { // over the old state. let mut deferred_axes: Option>)>> = initial_deferred_axes; + // A typed in-place rewrite lands on an entry the walk below will + // never look at: the mutation happened at the START path, not at a + // level the walk climbs through. Refresh that one row here, before + // the climb, and seed the same deferred state the walk uses so the + // indexed element one level up is rebuilt over the NEW secondary + // roots rather than re-read from stale root keys. + if let Some((changed_key, old_state)) = changed_indexed_entry + && child_tree.tree_type.is_indexed_primary() + { + let (container_path, indexed_key) = cost_return_on_error_no_add!( + cost, + current_path + .derive_parent() + .ok_or(Error::CorruptedCodeExecution( + "an indexed primary requires a parent holding its indexed element", + )) + ); + let indexed_element = { + let container = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + container_path, + transaction, + Some(batch), + grove_version, + ) + ); + cost_return_on_error!( + &mut cost, + Element::get(&container, indexed_key, true, grove_version) + .map_err(Error::MerkError) + ) + }; + + // Post-rewrite state, read from the primary the caller already + // updated. `None` means the rewrite removed the entry. + let new_state = { + let maybe = cost_return_on_error!( + &mut cost, + Element::get_optional(&child_tree, changed_key, true, grove_version) + .map_err(Error::MerkError) + ); + match maybe { + None => None, + Some(element) => { + let value_hash = cost_return_on_error!( + &mut cost, + child_tree + .get_value_hash( + changed_key, + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(Error::MerkError) + ); + let (count, sum) = element.count_sum_value_or_default(); + value_hash.map(|value_hash| { + crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, + } + }) + } + } + }; + + let axes: Vec<(u8, Option>)> = match indexed_element.underlying() { + Element::ProvableCountIndexedTree(_, secondary, ..) => vec![( + grovedb_element::indexed::IndexAxis::Count.tag(), + secondary.clone(), + )], + Element::ProvableSumIndexedTree(_, secondary, ..) => vec![( + grovedb_element::indexed::IndexAxis::Sum.tag(), + secondary.clone(), + )], + Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => axes.clone(), + _ => { + return Err(Error::CorruptedData( + "an indexed primary is not held by an indexed-tree element".to_string(), + )) + .wrap_with_cost(cost); + } + }; + let is_multi_axis = matches!( + indexed_element.underlying(), + Element::ProvableCountProvableSumIndexedTree(..) + ); + + let mut refreshed: Vec<(u8, Hash, Option>)> = Vec::with_capacity(axes.len()); + for (tag, secondary_root_key) in axes { + let axis = cost_return_on_error_no_add!( + cost, + grovedb_element::indexed::IndexAxis::try_from_tag(tag).map_err(|e| { + Error::CorruptedData(format!( + "invalid axis tag on an indexed element during row refresh: {e}" + )) + }) + ); + let mut secondary = cost_return_on_error!( + &mut cost, + self.open_indexed_secondary_at_path( + current_path.clone(), + axis, + secondary_root_key, + transaction, + Some(batch), + grove_version, + ) + ); + // Full old → new transition, NOT an in-place refresh: a + // caller-supplied element can carry different aggregates + // (`replace_subtree_root` does), which moves the sort key. + // Refreshing in place would strand the old row at the old + // key. + cost_return_on_error!( + &mut cost, + mirror_indexed_axis_to_secondary( + &mut secondary, + axis, + changed_key, + old_state, + new_state, + grove_version, + ) + ); + let (sec_hash, sec_key, _) = cost_return_on_error!( + &mut cost, + secondary + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) + ); + refreshed.push((tag, sec_hash, sec_key)); + } + + // Seed the slot this variant's element rebuild actually reads. + // The two are not interchangeable, and seeding the wrong one + // leaves state set for a later iteration with no indexed + // element to apply it to. + if is_multi_axis { + deferred_axes = Some(refreshed); + } else { + deferred_secondary = refreshed.first().map(|(_, hash, key)| (*hash, key.clone())); + } + } + while let Some((parent_path, parent_key)) = current_path.derive_parent() { let mut parent_tree: Merk = cost_return_on_error!( &mut cost, diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 25fd76588..5e324568a 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -146,33 +146,17 @@ impl GroveDb { ); // 5. Propagate changes - // Mirror the moved entry into the primary's axis secondaries, - // then propagate seeded with the mirror's per-axis roots so the - // indexed element one level up is rebuilt over the NEW secondary - // state rather than re-read from stale root keys. - let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( - &mut cost, - self.mirror_indexed_entry_and_seed( - path.clone(), - &parent_merk, - key, - old_indexed_state, - tx.as_ref(), - &batch, - grove_version, - ) - ); let mut merk_cache = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction_with_initial_deferred_axes( + self.propagate_changes_with_transaction_refreshing_indexed_row( merk_cache, path, - initial_deferred_secondary, - initial_deferred_axes, + key, + old_indexed_state, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 2ddee1c08..06383291f 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -231,33 +231,17 @@ impl GroveDb { ); // 6. Propagate changes from parent upward - // Mirror the moved entry into the primary's axis secondaries, - // then propagate seeded with the mirror's per-axis roots so the - // indexed element one level up is rebuilt over the NEW secondary - // state rather than re-read from stale root keys. - let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( - &mut cost, - self.mirror_indexed_entry_and_seed( - path.clone(), - &parent_merk, - key, - old_indexed_state, - tx.as_ref(), - &batch, - grove_version, - ) - ); let mut merk_cache = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction_with_initial_deferred_axes( + self.propagate_changes_with_transaction_refreshing_indexed_row( merk_cache, path, - initial_deferred_secondary, - initial_deferred_axes, + key, + old_indexed_state, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/dense_tree.rs b/grovedb/src/operations/dense_tree.rs index 84477ce11..431ec719a 100644 --- a/grovedb/src/operations/dense_tree.rs +++ b/grovedb/src/operations/dense_tree.rs @@ -143,22 +143,6 @@ impl GroveDb { ); // 5. Propagate changes - // Mirror the moved entry into the primary's axis secondaries, - // then propagate seeded with the mirror's per-axis roots so the - // indexed element one level up is rebuilt over the NEW secondary - // state rather than re-read from stale root keys. - let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( - &mut cost, - self.mirror_indexed_entry_and_seed( - path.clone(), - &parent_merk, - key, - old_indexed_state, - tx.as_ref(), - &batch, - grove_version, - ) - ); let mut merk_cache: HashMap, Merk> = HashMap::new(); @@ -166,11 +150,11 @@ impl GroveDb { cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction_with_initial_deferred_axes( + self.propagate_changes_with_transaction_refreshing_indexed_row( merk_cache, path, - initial_deferred_secondary, - initial_deferred_axes, + key, + old_indexed_state, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 1fd8dd990..273c07ad2 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -157,34 +157,17 @@ impl GroveDb { .map_err(|e| e.into()) ); - // 5. Mirror the moved entry into the primary's axis secondaries, - // then propagate upward seeded with the mirror's per-axis roots - // so the indexed element one level up is rebuilt over the NEW - // secondary state rather than re-read from stale root keys. - let (initial_deferred_secondary, initial_deferred_axes) = cost_return_on_error!( - &mut cost, - self.mirror_indexed_entry_and_seed( - path.clone(), - &parent_merk, - key, - old_indexed_state, - tx.as_ref(), - &batch, - grove_version, - ) - ); - let mut merk_cache: HashMap, Merk> = HashMap::new(); merk_cache.insert(path.clone(), parent_merk); cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction_with_initial_deferred_axes( + self.propagate_changes_with_transaction_refreshing_indexed_row( merk_cache, path, - initial_deferred_secondary, - initial_deferred_axes, + key, + old_indexed_state, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/operations/replace_subtree_root.rs b/grovedb/src/operations/replace_subtree_root.rs index 3395f71b3..3975aed2b 100644 --- a/grovedb/src/operations/replace_subtree_root.rs +++ b/grovedb/src/operations/replace_subtree_root.rs @@ -32,7 +32,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, CostResult, CostsExt, OperationCost, }; use grovedb_merk::element::{ - insert::ElementInsertToStorageExtensions, tree_type::ElementTreeTypeExtensions, + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + tree_type::ElementTreeTypeExtensions, }; use grovedb_path::SubtreePath; use grovedb_storage::{Storage, StorageBatch}; @@ -83,6 +84,22 @@ impl GroveDb { ) ); + // If the parent is an indexed primary, snapshot this entry BEFORE + // the rewrite. A canonical secondary row binds the entry's committed + // value hash, which this replaces outright — and unlike the non-Merk + // appends, `new_element` is caller-supplied, so its aggregates (and + // therefore the row's sort key) can differ from what is there. The + // old state is what lets the refresh move the row rather than strand + // it at the old key. + let old_element = cost_return_on_error!( + &mut cost, + Element::get(&parent_merk, key, true, grove_version).map_err(Error::MerkError) + ); + let old_indexed_state = cost_return_on_error!( + &mut cost, + GroveDb::capture_indexed_entry_state(&parent_merk, key, &old_element, grove_version) + ); + cost_return_on_error_into!( &mut cost, new_element.insert_subtree( @@ -99,9 +116,11 @@ impl GroveDb { cost_return_on_error!( &mut cost, - self.propagate_changes_with_transaction( + self.propagate_changes_with_transaction_refreshing_indexed_row( merk_cache, path, + key, + old_indexed_state, tx.as_ref(), &batch, grove_version, diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index a9e6d60f5..f8e2fc64f 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -25,6 +25,7 @@ mod tests { use crate::{ operations::indexed_tree::make_axis_secondary_key, + query_result_type::IndexedAxisEntrySliceExt, tests::{make_test_grovedb, TEST_LEAF}, Element, GroveDb, }; @@ -968,6 +969,141 @@ mod tests { ); } + /// `replace_subtree_root` is the fifth direct write path that rewrites + /// an entry in place, and the only one whose new element is + /// CALLER-SUPPLIED — so unlike the non-Merk appends, its aggregates can + /// differ from what was there, moving the row's sort key. + /// + /// A refresh that only rewrote the row at its existing key would strand + /// the old row at the old key. That is why the refresh takes the + /// pre-rewrite state and applies a full old → new transition. + /// + /// Feature-gated because the API is: it is the unsafe dump/load seam, + /// where the caller owns hash-vs-state correctness. + #[cfg(feature = "unsafe-dump-load")] + #[test] + fn replace_subtree_root_moves_the_row_when_aggregates_change() { + use crate::operations::indexed_tree::make_axis_secondary_key; + + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + // A count-bearing child, so its aggregate is what the row sorts on. + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"sub", + Element::empty_provable_count_tree(), + None, + grove_version, + ) + .unwrap() + .expect("count-tree child"); + for i in 0..3u8 { + db.insert( + [TEST_LEAF, b"cidx", b"sub"].as_ref(), + &[b'k', i], + Element::new_item(vec![i]), + None, + None, + grove_version, + ) + .unwrap() + .expect("populate"); + } + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!(issues.is_empty(), "setup reported: {issues:?}"); + + // The row currently sits at count 3. + let before = db + .indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) + .unwrap() + .expect("top_k"); + assert_eq!(before.key_pairs(), vec![(3u64, b"sub".to_vec())]); + + // Read the child's real root so the replacement is internally + // consistent, then re-state the element with a DIFFERENT count. + // That is the caller's prerogative on this API, and it is what + // moves the row. + let (child_root, child_root_key) = { + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let path_segments: [&[u8]; 3] = [TEST_LEAF, b"cidx".as_ref(), b"sub".as_ref()]; + let merk = db + .open_transactional_merk_at_path( + (&path_segments).into(), + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open child"); + let (hash, root_key, _) = merk + .root_hash_key_and_aggregate_data() + .unwrap() + .expect("child root"); + (hash, root_key) + }; + + db.replace_subtree_root( + [TEST_LEAF, b"cidx"].as_ref(), + b"sub", + Element::ProvableCountTree(child_root_key, 9, None), + child_root, + None, + grove_version, + ) + .unwrap() + .expect("replace_subtree_root under an indexed primary"); + + // The row must have MOVED to count 9 with nothing left behind at + // count 3. A stranded row would surface here as a SECOND entry: + // the axis read enumerates the secondary, so an orphan at the old + // key comes back alongside the new one. + let after = db + .indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) + .unwrap() + .expect("top_k"); + assert_eq!( + after.key_pairs(), + vec![(9u64, b"sub".to_vec())], + "the row must move to the new count, leaving nothing at the old one" + ); + + // `verify_grovedb` reports the child's own aggregate as + // inconsistent — deliberately, since this test states a count the + // subtree's contents do not support, which is exactly the + // hash-vs-state correctness `replace_subtree_root` hands to the + // caller. What must NOT appear is any indexed-row sentinel: the + // relationship between the primary entry and its secondary row is + // this code's responsibility, not the caller's. + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + let indexed_issues: Vec<_> = issues + .keys() + .filter(|p| { + p.iter() + .any(|seg| seg.starts_with(b"__cidx_") || seg.starts_with(b"__psit_")) + }) + .collect(); + assert!( + indexed_issues.is_empty(), + "replace_subtree_root left the indexed row inconsistent: {indexed_issues:?}" + ); + assert_ne!( + make_axis_secondary_key(IndexAxis::Count, 3, 0, b"sub"), + make_axis_secondary_key(IndexAxis::Count, 9, 0, b"sub"), + "test premise: the replacement really does move the sort key" + ); + } + /// Ordinary user references keep their own semantics. The one-hop /// immediate-node rule is dedicated indexed-tree behaviour, so it must /// not leak into how a normal reference elsewhere in the grove is From 3c4d77327dba4982ac50265762abfac48306c0f5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:18:52 +0700 Subject: [PATCH 07/18] fix: keep the verify-only build compiling (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's `cargo clippy --workspace --all-features -- -D warnings` was failing on four unresolved imports, and the cause was worse than a lint: this branch broke `--no-default-features --features verify` outright. That is the light-client build — no storage, no transactions — so a verify-only consumer could not compile the crate at all. Two things were reaching into `minimal`-gated code from modules that must survive without it: - `target_chain.rs` was entirely unconditional, but BUILDING a chain reads storage. Split it: the builder moves behind `minimal`, while `shape_commitment` / `authenticate_target_chain` stay unconditional. Authenticating a chain is pure arithmetic over bytes the proof already carries, which is exactly what a light client needs. - The axis verifier rebuilds the canonical row a proof claims, so it needs the row definition — which lived in the `minimal`-gated write path. Moved the pure helpers (`axis_row_reference`, `axis_payload_sum`, `make_axis_secondary_key`, `axis_sort_key_len`, `count_value_as_sum`, `INDEXED_SECONDARY_MAX_HOP`) into a new verify-available `indexed_axis::canonical_row`, re-exported from `indexed_tree` so every existing write-path caller is unchanged. The placement matters for the property, not just the build: a light client rebuilds the row from the SAME definition the mirror wrote with, which is what makes the check meaningful rather than a restatement of whatever the prover sent. `clippy --workspace --all-features -- -D warnings` passes; the verify-only build has zero errors; 2739 grovedb tests pass. --- grovedb/src/operations/indexed_tree.rs | 196 +------ .../proof/indexed_axis/canonical_row.rs | 196 +++++++ .../src/operations/proof/indexed_axis/mod.rs | 1 + .../proof/indexed_axis/target_chain.rs | 531 +++++++++--------- .../operations/proof/indexed_axis/verify.rs | 5 +- 5 files changed, 488 insertions(+), 441 deletions(-) create mode 100644 grovedb/src/operations/proof/indexed_axis/canonical_row.rs diff --git a/grovedb/src/operations/indexed_tree.rs b/grovedb/src/operations/indexed_tree.rs index c6579e523..1385db8c6 100644 --- a/grovedb/src/operations/indexed_tree.rs +++ b/grovedb/src/operations/indexed_tree.rs @@ -56,6 +56,15 @@ use crate::{ TransactionArg, }; +/// The canonical row definition lives in +/// [`crate::operations::proof::indexed_axis::canonical_row`] so a +/// verify-only build can reach it — a light client rebuilds the row a +/// proof claims from the same definition the mirror writes with. Re-exported +/// here because this is where the write paths look for it. +pub(crate) use crate::operations::proof::indexed_axis::canonical_row::{ + axis_row_reference, axis_sort_key_len, decode_axis_row_reference, make_axis_secondary_key, +}; + /// Per-axis Merk tree type to open the secondary with. /// /// Every axis uses the dual-aggregate @@ -91,46 +100,6 @@ pub(crate) fn axis_secondary_tree_type(axis: IndexAxis) -> TreeType { } } -/// The count-axis secondary stores each entry's `count_value` as its -/// sum item, and sum items are `i64`: a count above `i64::MAX` cannot -/// be mirrored faithfully, so it FAILS CLOSED rather than clamping — -/// a clamped total would silently lie through authenticated state. -/// Unreachable for any real tree (a count is bounded by the number of -/// elements), so the guard is a type-level seam, not a live limit. -#[inline] -pub(crate) fn count_value_as_sum(count: u64) -> Result { - i64::try_from(count).map_err(|_| { - Error::CorruptedData(format!( - "count value {count} exceeds i64::MAX and cannot be mirrored into the \ - count-axis secondary's sum aggregate" - )) - }) -} - -/// Hop budget stamped on every canonical indexed-secondary row. -/// -/// The row binds the IMMEDIATE primary node, not a terminal: its -/// committed value hash is -/// `combine_hash(H(canonical_reference_bytes), primary_node_value_hash)` -/// where `primary_node_value_hash` is whatever the primary Merk stores -/// for that key — a simple hash for an item, a layered/combined hash for -/// a tree or a nested reference. Binding the immediate node is what keeps -/// the invariant LOCAL and therefore mirror-maintainable: a mutation to -/// some distant terminal cannot staleness this row without also rewriting -/// the primary entry, which is the event the mirror is driven by. -/// -/// This is dedicated indexed-tree behaviour and is NOT a relaxation of -/// ordinary user-reference semantics — an ordinary `max_hop = 1` -/// reference pointing at another reference remains ill-formed and keeps -/// its existing diagnostics. Every consumer of an indexed row must select -/// the immediate-node rule explicitly (that is what -/// [`resolve_indexed_row_target`] and [`verify_indexed_axis_content`] do); -/// nothing may infer it from `max_reference_hop == 1` alone. -/// -/// [`resolve_indexed_row_target`]: crate::GroveDb::resolve_indexed_row_target -/// [`verify_indexed_axis_content`]: crate::GroveDb -pub(crate) const INDEXED_SECONDARY_MAX_HOP: grovedb_element::MaxReferenceHop = Some(1); - /// One primary entry's mirror-relevant state. /// /// The aggregates decide the row's sort key and carried sum; the value @@ -153,146 +122,6 @@ pub(crate) struct IndexedEntryState { pub(crate) value_hash: grovedb_merk::CryptoHash, } -/// The sum an axis's canonical row carries — the axis PAYLOAD sum, which -/// is not universally the primary's sum: -/// -/// - Count → `count_value_as_sum(count)`, so a band TOTAL over the count -/// axis stays one committed scalar (issue #806). A plain `Reference` -/// here would fold to `(1, 0)` and silently zero every band total. -/// - Sum / Avg → the primary entry's own sum. -/// -/// Every writer and every checker must agree on this one definition; a -/// divergent copy either false-flags healthy state or makes two entry -/// points commit different roots for identical writes (#809 audit). -/// -/// Fallible only through [`count_value_as_sum`]'s fail-closed guard. -#[inline] -pub(crate) fn axis_payload_sum(axis: IndexAxis, count: u64, sum: i64) -> Result { - Ok(match axis { - IndexAxis::Count => count_value_as_sum(count)?, - IndexAxis::Sum | IndexAxis::Avg => sum, - }) -} - -/// THE per-axis secondary row — the single definition every writer and -/// every checker uses. The sort KEY encodes the ordering value; the row -/// itself is a canonical one-hop reference back to the primary entry, -/// carrying the axis payload sum so the secondary's dual aggregates fold -/// to `(1, axis_payload_sum)`. -/// -/// All three axes share one element family: -/// `ReferenceWithSumItem(SiblingReference(item_key), Some(1), sum)`. -/// -/// The `SiblingReference` is interpreted against the row's LOGICAL -/// origin — the indexed primary's path — not against the derived storage -/// prefix the secondary physically lives under (which is not a GroveDB -/// path at all). See [`indexed_row_target_key`] for the decoding side and -/// [`crate::operations::indexed_tree`]'s module docs for the origin rule. -/// -/// Callers: the batch mirror row builder, the direct-path mirror, the -/// propagation mirror, `verify_grovedb`'s expected-row check, the axis -/// proof generator, and the average-case cost estimator's worst-case row. -/// -/// Fallible only through [`count_value_as_sum`]'s fail-closed guard. -pub(crate) fn axis_row_reference( - axis: IndexAxis, - item_key: &[u8], - count: u64, - sum: i64, -) -> Result { - Ok(Element::new_reference_with_sum_item_with_hops( - grovedb_element::reference_path::ReferencePathType::SiblingReference(item_key.to_vec()), - INDEXED_SECONDARY_MAX_HOP, - axis_payload_sum(axis, count, sum)?, - )) -} - -/// Decode a stored secondary row, enforcing the canonical shape and -/// returning `(target_item_key, carried_sum)`. -/// -/// Rejects anything that is not exactly -/// `ReferenceWithSumItem(SiblingReference(_), Some(1), _)` — including -/// the legacy placeholder payloads (`SumItem` / `ItemWithSumItem`), a -/// plain `Reference` (which would fold to `(1, 0)`), a non-sibling -/// reference type, and a wrong hop budget. `describe` labels the caller -/// in the error so a corruption report says where it was caught. -pub(crate) fn decode_axis_row_reference<'a>( - row: &'a Element, - describe: &str, -) -> Result<(&'a [u8], i64), Error> { - match row { - Element::ReferenceWithSumItem(reference_path, max_hop, sum, _) => { - if *max_hop != INDEXED_SECONDARY_MAX_HOP { - return Err(Error::CorruptedData(format!( - "{describe}: indexed secondary row carries max_reference_hop {max_hop:?}, \ - canonical rows are one-hop ({INDEXED_SECONDARY_MAX_HOP:?})" - ))); - } - match reference_path { - grovedb_element::reference_path::ReferencePathType::SiblingReference(key) => { - Ok((key.as_slice(), *sum)) - } - other => Err(Error::CorruptedData(format!( - "{describe}: indexed secondary row must be a SiblingReference to its \ - primary entry, found {other}" - ))), - } - } - other => Err(Error::CorruptedData(format!( - "{describe}: indexed secondary row must be ReferenceWithSumItem, found {}", - other.type_str() - ))), - } -} - -/// Build the secondary key bytes for an entry at `item_key` under the -/// given axis, given the relevant aggregate values: -/// - count axis → `count_be(8) ‖ item_key` -/// - sum axis → `sum_sortable_be(8) ‖ item_key` -/// - avg axis → `avg_sortable_be(16) ‖ item_key` -#[inline] -pub(crate) fn make_axis_secondary_key( - axis: IndexAxis, - count: u64, - sum: i64, - item_key: &[u8], -) -> Vec { - match axis { - IndexAxis::Count => { - let prefix = encode_count_sort_key(count); - let mut k = Vec::with_capacity(prefix.len() + item_key.len()); - k.extend_from_slice(&prefix); - k.extend_from_slice(item_key); - k - } - IndexAxis::Sum => { - let prefix = encode_sum_sort_key(sum); - let mut k = Vec::with_capacity(prefix.len() + item_key.len()); - k.extend_from_slice(&prefix); - k.extend_from_slice(item_key); - k - } - IndexAxis::Avg => { - let avg_fp = grovedb_element::indexed::compute_avg_fixed_point(sum, count); - let prefix = encode_avg_sort_key(avg_fp); - let mut k = Vec::with_capacity(prefix.len() + item_key.len()); - k.extend_from_slice(&prefix); - k.extend_from_slice(item_key); - k - } - } -} - -/// Width in bytes of an axis's sort-key prefix inside a secondary key -/// (`sort_key ‖ item_key`). -#[inline] -pub(crate) fn axis_sort_key_len(axis: IndexAxis) -> usize { - match axis { - IndexAxis::Count | IndexAxis::Sum => 8, - IndexAxis::Avg => 16, - } -} - /// Child-element rules specific to an indexed primary's variant. /// /// Merk's `validate_insertable_into` answers the generic question of whether an @@ -3451,8 +3280,9 @@ mod axis_row_reference_tests { use grovedb_element::{indexed::IndexAxis, reference_path::ReferencePathType}; use grovedb_version::version::GroveVersion; - use super::{ - axis_payload_sum, axis_row_reference, decode_axis_row_reference, INDEXED_SECONDARY_MAX_HOP, + use super::{axis_row_reference, decode_axis_row_reference}; + use crate::operations::proof::indexed_axis::canonical_row::{ + axis_payload_sum, INDEXED_SECONDARY_MAX_HOP, }; use crate::Element; @@ -3623,7 +3453,7 @@ mod count_value_as_sum_tests { //! because a clamped value would flow into hash-bound authenticated //! state as a silently wrong total. - use super::count_value_as_sum; + use crate::operations::proof::indexed_axis::canonical_row::count_value_as_sum; #[test] fn converts_in_domain_and_fails_closed_above_i64_max() { diff --git a/grovedb/src/operations/proof/indexed_axis/canonical_row.rs b/grovedb/src/operations/proof/indexed_axis/canonical_row.rs new file mode 100644 index 000000000..fce72b74b --- /dev/null +++ b/grovedb/src/operations/proof/indexed_axis/canonical_row.rs @@ -0,0 +1,196 @@ +//! The canonical indexed-secondary row: its shape, its sort key, and the +//! encode/decode pair every writer and every checker shares. +//! +//! This module is deliberately free of storage and transaction types so it +//! compiles in a **verify-only** build. A light client that never opens a +//! Merk still has to rebuild the canonical row a proof claims, and it +//! rebuilds it from exactly the same definition the mirror wrote with — +//! which is the property that makes the check meaningful rather than a +//! restatement of whatever the prover sent. + +use grovedb_element::indexed::{ + encode_avg_sort_key, encode_count_sort_key, encode_sum_sort_key, IndexAxis, +}; + +use crate::{Element, Error}; + +/// The count-axis secondary stores each entry's `count_value` as its +/// sum item, and sum items are `i64`: a count above `i64::MAX` cannot +/// be mirrored faithfully, so it FAILS CLOSED rather than clamping — +/// a clamped total would silently lie through authenticated state. +/// Unreachable for any real tree (a count is bounded by the number of +/// elements), so the guard is a type-level seam, not a live limit. +#[inline] +pub(crate) fn count_value_as_sum(count: u64) -> Result { + i64::try_from(count).map_err(|_| { + Error::CorruptedData(format!( + "count value {count} exceeds i64::MAX and cannot be mirrored into the \ + count-axis secondary's sum aggregate" + )) + }) +} + +/// Hop budget stamped on every canonical indexed-secondary row. +/// +/// The row binds the IMMEDIATE primary node, not a terminal: its +/// committed value hash is +/// `combine_hash(H(canonical_reference_bytes), primary_node_value_hash)` +/// where `primary_node_value_hash` is whatever the primary Merk stores +/// for that key — a simple hash for an item, a layered/combined hash for +/// a tree or a nested reference. Binding the immediate node is what keeps +/// the invariant LOCAL and therefore mirror-maintainable: a mutation to +/// some distant terminal cannot staleness this row without also rewriting +/// the primary entry, which is the event the mirror is driven by. +/// +/// This is dedicated indexed-tree behaviour and is NOT a relaxation of +/// ordinary user-reference semantics — an ordinary `max_hop = 1` +/// reference pointing at another reference remains ill-formed and keeps +/// its existing diagnostics. Every consumer of an indexed row must select +/// the immediate-node rule explicitly (that is what +/// [`resolve_indexed_row_target`] and [`verify_indexed_axis_content`] do); +/// nothing may infer it from `max_reference_hop == 1` alone. +/// +/// [`resolve_indexed_row_target`]: crate::GroveDb::resolve_indexed_row_target +/// [`verify_indexed_axis_content`]: crate::GroveDb +pub(crate) const INDEXED_SECONDARY_MAX_HOP: grovedb_element::MaxReferenceHop = Some(1); + +/// The sum an axis's canonical row carries — the axis PAYLOAD sum, which +/// is not universally the primary's sum: +/// +/// - Count → `count_value_as_sum(count)`, so a band TOTAL over the count +/// axis stays one committed scalar (issue #806). A plain `Reference` +/// here would fold to `(1, 0)` and silently zero every band total. +/// - Sum / Avg → the primary entry's own sum. +/// +/// Every writer and every checker must agree on this one definition; a +/// divergent copy either false-flags healthy state or makes two entry +/// points commit different roots for identical writes (#809 audit). +/// +/// Fallible only through [`count_value_as_sum`]'s fail-closed guard. +#[inline] +pub(crate) fn axis_payload_sum(axis: IndexAxis, count: u64, sum: i64) -> Result { + Ok(match axis { + IndexAxis::Count => count_value_as_sum(count)?, + IndexAxis::Sum | IndexAxis::Avg => sum, + }) +} + +/// THE per-axis secondary row — the single definition every writer and +/// every checker uses. The sort KEY encodes the ordering value; the row +/// itself is a canonical one-hop reference back to the primary entry, +/// carrying the axis payload sum so the secondary's dual aggregates fold +/// to `(1, axis_payload_sum)`. +/// +/// All three axes share one element family: +/// `ReferenceWithSumItem(SiblingReference(item_key), Some(1), sum)`. +/// +/// The `SiblingReference` is interpreted against the row's LOGICAL +/// origin — the indexed primary's path — not against the derived storage +/// prefix the secondary physically lives under (which is not a GroveDB +/// path at all). See [`indexed_row_target_key`] for the decoding side and +/// [`crate::operations::indexed_tree`]'s module docs for the origin rule. +/// +/// Callers: the batch mirror row builder, the direct-path mirror, the +/// propagation mirror, `verify_grovedb`'s expected-row check, the axis +/// proof generator, and the average-case cost estimator's worst-case row. +/// +/// Fallible only through [`count_value_as_sum`]'s fail-closed guard. +pub(crate) fn axis_row_reference( + axis: IndexAxis, + item_key: &[u8], + count: u64, + sum: i64, +) -> Result { + Ok(Element::new_reference_with_sum_item_with_hops( + grovedb_element::reference_path::ReferencePathType::SiblingReference(item_key.to_vec()), + INDEXED_SECONDARY_MAX_HOP, + axis_payload_sum(axis, count, sum)?, + )) +} + +/// Decode a stored secondary row, enforcing the canonical shape and +/// returning `(target_item_key, carried_sum)`. +/// +/// Rejects anything that is not exactly +/// `ReferenceWithSumItem(SiblingReference(_), Some(1), _)` — including +/// the legacy placeholder payloads (`SumItem` / `ItemWithSumItem`), a +/// plain `Reference` (which would fold to `(1, 0)`), a non-sibling +/// reference type, and a wrong hop budget. `describe` labels the caller +/// in the error so a corruption report says where it was caught. +#[cfg(feature = "minimal")] +pub(crate) fn decode_axis_row_reference<'a>( + row: &'a Element, + describe: &str, +) -> Result<(&'a [u8], i64), Error> { + match row { + Element::ReferenceWithSumItem(reference_path, max_hop, sum, _) => { + if *max_hop != INDEXED_SECONDARY_MAX_HOP { + return Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row carries max_reference_hop {max_hop:?}, \ + canonical rows are one-hop ({INDEXED_SECONDARY_MAX_HOP:?})" + ))); + } + match reference_path { + grovedb_element::reference_path::ReferencePathType::SiblingReference(key) => { + Ok((key.as_slice(), *sum)) + } + other => Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row must be a SiblingReference to its \ + primary entry, found {other}" + ))), + } + } + other => Err(Error::CorruptedData(format!( + "{describe}: indexed secondary row must be ReferenceWithSumItem, found {}", + other.type_str() + ))), + } +} + +/// Build the secondary key bytes for an entry at `item_key` under the +/// given axis, given the relevant aggregate values: +/// - count axis → `count_be(8) ‖ item_key` +/// - sum axis → `sum_sortable_be(8) ‖ item_key` +/// - avg axis → `avg_sortable_be(16) ‖ item_key` +#[inline] +pub(crate) fn make_axis_secondary_key( + axis: IndexAxis, + count: u64, + sum: i64, + item_key: &[u8], +) -> Vec { + match axis { + IndexAxis::Count => { + let prefix = encode_count_sort_key(count); + let mut k = Vec::with_capacity(prefix.len() + item_key.len()); + k.extend_from_slice(&prefix); + k.extend_from_slice(item_key); + k + } + IndexAxis::Sum => { + let prefix = encode_sum_sort_key(sum); + let mut k = Vec::with_capacity(prefix.len() + item_key.len()); + k.extend_from_slice(&prefix); + k.extend_from_slice(item_key); + k + } + IndexAxis::Avg => { + let avg_fp = grovedb_element::indexed::compute_avg_fixed_point(sum, count); + let prefix = encode_avg_sort_key(avg_fp); + let mut k = Vec::with_capacity(prefix.len() + item_key.len()); + k.extend_from_slice(&prefix); + k.extend_from_slice(item_key); + k + } + } +} + +/// Width in bytes of an axis's sort-key prefix inside a secondary key +/// (`sort_key ‖ item_key`). +#[inline] +pub(crate) fn axis_sort_key_len(axis: IndexAxis) -> usize { + match axis { + IndexAxis::Count | IndexAxis::Sum => 8, + IndexAxis::Avg => 16, + } +} diff --git a/grovedb/src/operations/proof/indexed_axis/mod.rs b/grovedb/src/operations/proof/indexed_axis/mod.rs index f5c40113c..30f207651 100644 --- a/grovedb/src/operations/proof/indexed_axis/mod.rs +++ b/grovedb/src/operations/proof/indexed_axis/mod.rs @@ -62,6 +62,7 @@ //! shape only when the range really is out of the axis's domain. mod axis_api; +pub(crate) mod canonical_row; mod envelope; #[cfg(feature = "minimal")] mod generate; diff --git a/grovedb/src/operations/proof/indexed_axis/target_chain.rs b/grovedb/src/operations/proof/indexed_axis/target_chain.rs index f0ffe0745..9c8443f33 100644 --- a/grovedb/src/operations/proof/indexed_axis/target_chain.rs +++ b/grovedb/src/operations/proof/indexed_axis/target_chain.rs @@ -9,11 +9,10 @@ //! # Why this carries no path proofs //! //! Each chain entry's commitment is reconstructed from its own bytes plus -//! the NEXT entry's commitment, and the head's commitment is what the row -//! binds. The row's hash is bound into the secondary root, which the axis -//! verifier binds to the indexed element, which chains to the grove root -//! — so substituting any value anywhere in the chain moves the root and -//! is rejected. +//! the terminal's, and the head's commitment is what the row binds. The +//! row's hash is bound into the secondary root, which the axis verifier +//! binds to the indexed element, which chains to the grove root — so +//! substituting any value in the chain moves the root and is rejected. //! //! This is the same trust model shipped GroveDB reference proofs already //! use: `KVRefValueHash*` binds a reference's committed target hash to the @@ -25,106 +24,88 @@ //! A row whose committed target hash disagrees with what actually lives //! at the referenced path is stale state, not an unsound proof — and //! `verify_grovedb`'s stale-target-hash check is what detects that. +//! +//! # Feature split +//! +//! BUILDING a chain reads storage, so it needs `minimal`. AUTHENTICATING +//! one is pure arithmetic over bytes the proof already carries, so it +//! compiles in a verify-only build — which is the whole point: a light +//! client with no Merk must still be able to check a chain. -use grovedb_costs::{ - cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, -}; -use grovedb_element::indexed::IndexAxis; use grovedb_merk::{ - element::get::ElementFetchFromStorageExtensions, tree::{axes_digest, combine_hash, combine_hash_three, value_hash}, CryptoHash, }; -use grovedb_path::{SubtreePath, SubtreePathBuilder}; -use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; use super::{IndexedTargetChain, IndexedTargetCommitment, IndexedTargetNode}; -use crate::{ - operations::MAX_REFERENCE_HOPS, reference_path::path_from_reference_path_type, Element, Error, - GroveDb, Transaction, -}; +use crate::{Element, Error}; -/// Derive the commitment shape for one node, reading whatever roots that -/// shape folds in. -fn build_commitment<'db>( - grovedb: &'db GroveDb, - element: &Element, - qualified_path: &[Vec], - transaction: &'db Transaction, - batch: &'db StorageBatch, - grove_version: &GroveVersion, -) -> CostResult { - let mut cost = OperationCost::default(); - let underlying = element.underlying(); +#[cfg(feature = "minimal")] +mod build { + use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, + }; + use grovedb_element::indexed::IndexAxis; + use grovedb_merk::{element::get::ElementFetchFromStorageExtensions, tree::CryptoHash}; + use grovedb_path::{SubtreePath, SubtreePathBuilder}; + use grovedb_storage::StorageBatch; + use grovedb_version::version::GroveVersion; - if underlying.is_reference() { - return Ok(IndexedTargetCommitment::Reference).wrap_with_cost(cost); - } - if !underlying.is_any_tree() { - return Ok(IndexedTargetCommitment::Simple).wrap_with_cost(cost); - } + use super::super::{IndexedTargetChain, IndexedTargetCommitment, IndexedTargetNode}; + use crate::{ + operations::MAX_REFERENCE_HOPS, reference_path::path_from_reference_path_type, Element, + Error, GroveDb, Transaction, + }; - let path_owned: SubtreePathBuilder> = - SubtreePathBuilder::owned_from_iter(qualified_path.iter().cloned()); - let node_path = SubtreePath::from(&path_owned); + /// Derive the commitment shape for one node, reading whatever roots that + /// shape folds in. + fn build_commitment<'db>( + grovedb: &'db GroveDb, + element: &Element, + qualified_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + let underlying = element.underlying(); - // Read the primary/inner root every tree-ish shape folds in. - let inner_root = |cost: &mut OperationCost| -> Result { - let merk = grovedb - .open_transactional_merk_at_path( - node_path.clone(), - transaction, - Some(batch), - grove_version, - ) - .unwrap_add_cost(cost)?; - merk.root_hash_key_and_aggregate_data() - .unwrap_add_cost(cost) - .map(|(hash, ..)| hash) - .map_err(Error::MerkError) - }; + if underlying.is_reference() { + return Ok(IndexedTargetCommitment::Reference).wrap_with_cost(cost); + } + if !underlying.is_any_tree() { + return Ok(IndexedTargetCommitment::Simple).wrap_with_cost(cost); + } - let commitment = match underlying { - Element::ProvableCountIndexedTree(_, secondary_root_key, ..) - | Element::ProvableSumIndexedTree(_, secondary_root_key, ..) => { - let axis = match underlying { - Element::ProvableCountIndexedTree(..) => IndexAxis::Count, - _ => IndexAxis::Sum, - }; - let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); - let secondary = cost_return_on_error!( - &mut cost, - grovedb.open_indexed_secondary_at_path( + let path_owned: SubtreePathBuilder> = + SubtreePathBuilder::owned_from_iter(qualified_path.iter().cloned()); + let node_path = SubtreePath::from(&path_owned); + + // Read the primary/inner root every tree-ish shape folds in. + let inner_root = |cost: &mut OperationCost| -> Result { + let merk = grovedb + .open_transactional_merk_at_path( node_path.clone(), - axis, - secondary_root_key.clone(), transaction, Some(batch), grove_version, ) - ); - let (secondary_root_hash, ..) = cost_return_on_error!( - &mut cost, - secondary - .root_hash_key_and_aggregate_data() - .map_err(Error::MerkError) - ); - IndexedTargetCommitment::IndexedSingle { - primary_root_hash, - secondary_root_hash, - } - } - Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => { - let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); - let mut axis_hashes: Vec<(u8, CryptoHash)> = Vec::with_capacity(axes.len()); - for (tag, secondary_root_key) in axes { - let axis = cost_return_on_error_no_add!( - cost, - IndexAxis::try_from_tag(*tag).map_err(|e| Error::CorruptedData(format!( - "indexed target chain: invalid PCPSIT axis tag: {e}" - ))) - ); + .unwrap_add_cost(cost)?; + merk.root_hash_key_and_aggregate_data() + .unwrap_add_cost(cost) + .map(|(hash, ..)| hash) + .map_err(Error::MerkError) + }; + + let commitment = match underlying { + Element::ProvableCountIndexedTree(_, secondary_root_key, ..) + | Element::ProvableSumIndexedTree(_, secondary_root_key, ..) => { + let axis = match underlying { + Element::ProvableCountIndexedTree(..) => IndexAxis::Count, + _ => IndexAxis::Sum, + }; + let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); let secondary = cost_return_on_error!( &mut cost, grovedb.open_indexed_secondary_at_path( @@ -142,205 +123,241 @@ fn build_commitment<'db>( .root_hash_key_and_aggregate_data() .map_err(Error::MerkError) ); - axis_hashes.push((*tag, secondary_root_hash)); + IndexedTargetCommitment::IndexedSingle { + primary_root_hash, + secondary_root_hash, + } } - IndexedTargetCommitment::IndexedMulti { - primary_root_hash, - axes: axis_hashes, + Element::ProvableCountProvableSumIndexedTree(_, _, _, axes, _) => { + let primary_root_hash = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); + let mut axis_hashes: Vec<(u8, CryptoHash)> = Vec::with_capacity(axes.len()); + for (tag, secondary_root_key) in axes { + let axis = cost_return_on_error_no_add!( + cost, + IndexAxis::try_from_tag(*tag).map_err(|e| Error::CorruptedData(format!( + "indexed target chain: invalid PCPSIT axis tag: {e}" + ))) + ); + let secondary = cost_return_on_error!( + &mut cost, + grovedb.open_indexed_secondary_at_path( + node_path.clone(), + axis, + secondary_root_key.clone(), + transaction, + Some(batch), + grove_version, + ) + ); + let (secondary_root_hash, ..) = cost_return_on_error!( + &mut cost, + secondary + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) + ); + axis_hashes.push((*tag, secondary_root_hash)); + } + IndexedTargetCommitment::IndexedMulti { + primary_root_hash, + axes: axis_hashes, + } } - } - // Every other tree-ish shape — ordinary/sum/count Merk trees and - // the non-Merk append trees — commits as - // `combine_hash(H(value), child_root)`. Non-Merk trees keep their - // state root in the same position, so they need no special case. - _ => { - let child_root = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); - IndexedTargetCommitment::Layered(child_root) - } - }; - Ok(commitment).wrap_with_cost(cost) -} - -/// Build the chain for one secondary row. -/// -/// The chain is at most TWO entries: the immediate primary entry, and — -/// only when that entry is itself a reference — the TERMINAL it resolves -/// to. Intermediate hops are deliberately not carried, because nothing -/// binds them: a GroveDB reference commits its terminal's value hash -/// directly (`follow_reference_get_value_hash` recurses past every -/// intermediate reference before the hash is baked into -/// `PutCombinedReference`), so the head's commitment reaches the terminal -/// in one step no matter how many hops the path takes. Carrying the -/// middle of the chain would hand a verifier bytes it cannot check. -pub(crate) fn build_target_chain<'db>( - grovedb: &'db GroveDb, - indexed_path: &[Vec], - primary_key: &[u8], - transaction: &'db Transaction, - batch: &'db StorageBatch, - grove_version: &GroveVersion, -) -> CostResult { - let mut cost = OperationCost::default(); - - // The head: the primary entry itself. - let head_parent: Vec<&[u8]> = indexed_path.iter().map(Vec::as_slice).collect(); - let head_merk = cost_return_on_error!( - &mut cost, - grovedb.open_transactional_merk_at_path( - head_parent.as_slice().into(), - transaction, - Some(batch), - grove_version, - ) - ); - let head_element = cost_return_on_error!( - &mut cost, - Element::get(&head_merk, primary_key, true, grove_version).map_err(|e| { - Error::CorruptedData(format!( - "indexed target chain: primary entry {} is missing: {e}", - hex::encode(primary_key) - )) - }) - ); - drop(head_merk); - - let mut head_qualified = indexed_path.to_vec(); - head_qualified.push(primary_key.to_vec()); - let head = cost_return_on_error!( - &mut cost, - build_chain_node( - grovedb, - &head_element, - &head_qualified, - transaction, - batch, - grove_version, - ) - ); - let mut nodes = vec![head]; - - // If the primary entry is a reference, walk to the terminal. Only the - // terminal is recorded — see this function's doc for why. - let mut current_element = head_element; - let mut current_parent = indexed_path.to_vec(); - let mut current_key = primary_key.to_vec(); - let mut visited = std::collections::HashSet::new(); - visited.insert(head_qualified); - - for _ in 0..=MAX_REFERENCE_HOPS { - let reference_path = match current_element.underlying() { - Element::Reference(reference_path, ..) - | Element::ReferenceWithSumItem(reference_path, ..) => reference_path.clone(), - // Not a reference: the head was the terminal, or we just - // reached it. + // Every other tree-ish shape — ordinary/sum/count Merk trees and + // the non-Merk append trees — commits as + // `combine_hash(H(value), child_root)`. Non-Merk trees keep their + // state root in the same position, so they need no special case. _ => { - if nodes.len() > 1 || !nodes[0].is_reference() { - return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); - } - // Unreachable: a reference head always takes the branch - // below at least once before we get here. - return Err(Error::CorruptedCodeExecution( - "indexed target chain: reference head produced no terminal", - )) - .wrap_with_cost(cost); + let child_root = cost_return_on_error_no_add!(cost, inner_root(&mut cost)); + IndexedTargetCommitment::Layered(child_root) } }; + Ok(commitment).wrap_with_cost(cost) + } - // `current_parent` — NOT the node's own qualified path. A relative - // reference resolves against its PARENT (`SiblingReference` - // appends its key to what it is given), so passing the node's own - // path would look for a child underneath the entry itself. - let next_qualified = match path_from_reference_path_type( - reference_path, - ¤t_parent, - Some(current_key.as_slice()), - ) { - Ok(p) => p, - Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), - }; - if !visited.insert(next_qualified.clone()) { - return Err(Error::CyclicReference).wrap_with_cost(cost); - } - let Some((next_key, next_parent)) = next_qualified.split_last() else { - return Err(Error::CorruptedPath( - "indexed target chain resolved an empty path".to_string(), - )) - .wrap_with_cost(cost); - }; - let next_parent_slices: Vec<&[u8]> = next_parent.iter().map(Vec::as_slice).collect(); - let next_merk = cost_return_on_error!( + /// Build the chain for one secondary row. + /// + /// The chain is at most TWO entries: the immediate primary entry, and — + /// only when that entry is itself a reference — the TERMINAL it resolves + /// to. Intermediate hops are deliberately not carried, because nothing + /// binds them: a GroveDB reference commits its terminal's value hash + /// directly (`follow_reference_get_value_hash` recurses past every + /// intermediate reference before the hash is baked into + /// `PutCombinedReference`), so the head's commitment reaches the terminal + /// in one step no matter how many hops the path takes. Carrying the + /// middle of the chain would hand a verifier bytes it cannot check. + pub(crate) fn build_target_chain<'db>( + grovedb: &'db GroveDb, + indexed_path: &[Vec], + primary_key: &[u8], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + + // The head: the primary entry itself. + let head_parent: Vec<&[u8]> = indexed_path.iter().map(Vec::as_slice).collect(); + let head_merk = cost_return_on_error!( &mut cost, grovedb.open_transactional_merk_at_path( - next_parent_slices.as_slice().into(), + head_parent.as_slice().into(), transaction, Some(batch), grove_version, ) ); - let next_element = cost_return_on_error!( + let head_element = cost_return_on_error!( &mut cost, - Element::get(&next_merk, next_key, true, grove_version).map_err(|e| { - Error::CorruptedReferencePathKeyNotFound(format!( - "indexed target chain: reference target {} is missing: {e}", - hex::encode(next_key) + Element::get(&head_merk, primary_key, true, grove_version).map_err(|e| { + Error::CorruptedData(format!( + "indexed target chain: primary entry {} is missing: {e}", + hex::encode(primary_key) )) }) ); - drop(next_merk); + drop(head_merk); + + let mut head_qualified = indexed_path.to_vec(); + head_qualified.push(primary_key.to_vec()); + let head = cost_return_on_error!( + &mut cost, + build_chain_node( + grovedb, + &head_element, + &head_qualified, + transaction, + batch, + grove_version, + ) + ); + let mut nodes = vec![head]; + + // If the primary entry is a reference, walk to the terminal. Only the + // terminal is recorded — see this function's doc for why. + let mut current_element = head_element; + let mut current_parent = indexed_path.to_vec(); + let mut current_key = primary_key.to_vec(); + let mut visited = std::collections::HashSet::new(); + visited.insert(head_qualified); + + for _ in 0..=MAX_REFERENCE_HOPS { + let reference_path = match current_element.underlying() { + Element::Reference(reference_path, ..) + | Element::ReferenceWithSumItem(reference_path, ..) => reference_path.clone(), + // Not a reference: the head was the terminal, or we just + // reached it. + _ => { + if nodes.len() > 1 || !nodes[0].is_reference() { + return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); + } + // Unreachable: a reference head always takes the branch + // below at least once before we get here. + return Err(Error::CorruptedCodeExecution( + "indexed target chain: reference head produced no terminal", + )) + .wrap_with_cost(cost); + } + }; - if !next_element.underlying().is_reference() { - let terminal = cost_return_on_error!( + // `current_parent` — NOT the node's own qualified path. A relative + // reference resolves against its PARENT (`SiblingReference` + // appends its key to what it is given), so passing the node's own + // path would look for a child underneath the entry itself. + let next_qualified = match path_from_reference_path_type( + reference_path, + ¤t_parent, + Some(current_key.as_slice()), + ) { + Ok(p) => p, + Err(e) => return Err(Error::from(e)).wrap_with_cost(cost), + }; + if !visited.insert(next_qualified.clone()) { + return Err(Error::CyclicReference).wrap_with_cost(cost); + } + let Some((next_key, next_parent)) = next_qualified.split_last() else { + return Err(Error::CorruptedPath( + "indexed target chain resolved an empty path".to_string(), + )) + .wrap_with_cost(cost); + }; + let next_parent_slices: Vec<&[u8]> = next_parent.iter().map(Vec::as_slice).collect(); + let next_merk = cost_return_on_error!( &mut cost, - build_chain_node( - grovedb, - &next_element, - &next_qualified, + grovedb.open_transactional_merk_at_path( + next_parent_slices.as_slice().into(), transaction, - batch, + Some(batch), grove_version, ) ); - nodes.push(terminal); - return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); + let next_element = cost_return_on_error!( + &mut cost, + Element::get(&next_merk, next_key, true, grove_version).map_err(|e| { + Error::CorruptedReferencePathKeyNotFound(format!( + "indexed target chain: reference target {} is missing: {e}", + hex::encode(next_key) + )) + }) + ); + drop(next_merk); + + if !next_element.underlying().is_reference() { + let terminal = cost_return_on_error!( + &mut cost, + build_chain_node( + grovedb, + &next_element, + &next_qualified, + transaction, + batch, + grove_version, + ) + ); + nodes.push(terminal); + return Ok(IndexedTargetChain { nodes }).wrap_with_cost(cost); + } + + current_parent = next_parent.to_vec(); + current_key = next_key.clone(); + current_element = next_element; } - current_parent = next_parent.to_vec(); - current_key = next_key.clone(); - current_element = next_element; + Err(Error::ReferenceLimit).wrap_with_cost(cost) } - Err(Error::ReferenceLimit).wrap_with_cost(cost) + /// Serialize one element and derive its commitment shape. + fn build_chain_node<'db>( + grovedb: &'db GroveDb, + element: &Element, + qualified_path: &[Vec], + transaction: &'db Transaction, + batch: &'db StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + let value = cost_return_on_error_no_add!( + cost, + element.serialize(grove_version).map_err(|e| { + Error::CorruptedData(format!("indexed target chain: serializing entry: {e}")) + }) + ); + let commitment = cost_return_on_error!( + &mut cost, + build_commitment( + grovedb, + element, + qualified_path, + transaction, + batch, + grove_version, + ) + ); + Ok(IndexedTargetNode { value, commitment }).wrap_with_cost(cost) + } } -/// Serialize one element and derive its commitment shape. -fn build_chain_node<'db>( - grovedb: &'db GroveDb, - element: &Element, - qualified_path: &[Vec], - transaction: &'db Transaction, - batch: &'db StorageBatch, - grove_version: &GroveVersion, -) -> CostResult { - let mut cost = OperationCost::default(); - let value = cost_return_on_error_no_add!( - cost, - element.serialize(grove_version).map_err(|e| { - Error::CorruptedData(format!("indexed target chain: serializing entry: {e}")) - }) - ); - let commitment = cost_return_on_error!( - &mut cost, - build_commitment( - grovedb, - element, - qualified_path, - transaction, - batch, - grove_version, - ) - ); - Ok(IndexedTargetNode { value, commitment }).wrap_with_cost(cost) -} +#[cfg(feature = "minimal")] +pub(crate) use build::build_target_chain; /// The commitment a node's own shape implies, for every shape except /// `Reference` (whose commitment needs the terminal and is folded by the diff --git a/grovedb/src/operations/proof/indexed_axis/verify.rs b/grovedb/src/operations/proof/indexed_axis/verify.rs index f85e06066..2893655c0 100644 --- a/grovedb/src/operations/proof/indexed_axis/verify.rs +++ b/grovedb/src/operations/proof/indexed_axis/verify.rs @@ -867,7 +867,10 @@ fn authenticate_axis_row( ) -> Result<(Vec, Element), Error> { use grovedb_merk::tree::{combine_hash, value_hash}; - use crate::operations::indexed_tree::{ + // From the verify-available canonical-row module, NOT the + // `minimal`-gated write path: a light client rebuilds the row a proof + // claims without ever opening a Merk. + use super::canonical_row::{ axis_payload_sum, axis_row_reference, axis_sort_key_len, make_axis_secondary_key, }; From 807d6d54ffc2b8395cd514481f381a924dfdf89a Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:33:58 +0700 Subject: [PATCH 08/18] =?UTF-8?q?fix:=20address=20CodeRabbit=20review=20?= =?UTF-8?q?=E2=80=94=20read=20consistency,=20cost=20sizing,=20hop=20bound?= =?UTF-8?q?=20(#814)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven valid findings from the review. Several others referenced code this branch has since deleted (`reference_resolution.rs`) or reverted (the merk `Node` variant), so they no longer apply. **Corruption was being read as absence, in three places.** All three built `Option` with `value_hash.map(...)`, collapsing "the entry does not exist" and "the entry exists but its node is unreachable from the committed root" into the same `None`. On the new side that hands the mirror `None` and DELETES a live row; on the old side it skips the delete of a row that moved. Both now fail loudly, matching what the propagation path already did for the identical condition. **Indexed reads used two snapshots.** `resolve_axis_entries` built its own `TxRef`, so with `transaction: None` the primary resolution ran under a different snapshot than the secondary scan that produced the rows. A commit in between could pair a stale row with a newer primary value, or report a primary the row still names as corrupted. It now takes the caller's transaction and passes it to `follow_reference` too. **The secondary layer was described as `AllItems`.** Its rows are `ReferenceWithSumItem`, and the two variants carry different element overheads (+3 vs +15), so every row was under-charged by 12 bytes. `added_bytes` is the one dimension a storage-fee reservation must never come in under. **A reference in a `ProvableCountSumTree` hard-errored.** That host is eligible for count-offset pagination but commits only the count into its node hash, so its feature type is `ProvableCountedSummedMerkNode` — which the post-pass did not match. It now takes the count-only node, the same variant `emit_returned_node` picks for that host's directly-valued rows. Mutation-checked: reverting the arm makes the new test fail with the original error. **The chain builder allowed one hop more than `follow_reference`.** `0..=MAX_REFERENCE_HOPS` let the prover build a chain `db.get` would refuse. Now `0..`. Also: the paginated verify path now runs the layer binding before row decoding, matching the range path, so both name the same defect for the same forgery; `assert_only_issue` asserts row-sentinel exclusivity (scoped to `__cidx_*`, since damaging a row legitimately moves the element's H1-A binding too); the commitment-tree append — the one non-Merk append live on mainnet — is now covered alongside the other three; and the book's verified-result type is corrected to `AxisEntries`. 43 targets, 2740 grovedb tests (2743 with `unsafe-dump-load`). `clippy --workspace --all-features -- -D warnings` passes; verify-only build clean. --- docs/book/src/count-indexed-tree.md | 3 +- .../src/estimated_costs/average_case_costs.rs | 8 ++- grovedb/src/lib.rs | 62 +++++++++++------ grovedb/src/operations/indexed_tree.rs | 19 +++--- grovedb/src/operations/proof/generate.rs | 22 ++++-- .../proof/indexed_axis/target_chain.rs | 6 +- .../operations/proof/indexed_axis/verify.rs | 23 ++++--- .../src/tests/count_offset_paginated_tests.rs | 67 +++++++++++++++++++ .../src/tests/indexed_reference_row_tests.rs | 49 ++++++++++++++ 9 files changed, 214 insertions(+), 45 deletions(-) diff --git a/docs/book/src/count-indexed-tree.md b/docs/book/src/count-indexed-tree.md index d3944ce32..0993bc0d0 100644 --- a/docs/book/src/count-indexed-tree.md +++ b/docs/book/src/count-indexed-tree.md @@ -488,7 +488,8 @@ let proof_bytes = db .prove_indexed_count_top_k(path, k, /* descending: */ true, transaction, grove_version)? .expect("prove"); let result = GroveDb::verify_indexed_count_top_k(&proof_bytes, &path, k)?; -// result.entries: Vec>, result.root_hash: [u8; 32] +// result.entries: AxisEntries::Count(Vec>) +// result.root_hash: [u8; 32] ``` The query returns `IndexedAxisEntry` rows — the count, the primary key, diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 290fd7cb6..6146c3e8a 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -540,7 +540,13 @@ impl GroveDb { tree_type: axis_secondary_tree_type(*axis), // 1:1 with the primary. estimated_layer_count: primary_layer_information.estimated_layer_count, - estimated_layer_sizes: EstimatedLayerSizes::AllItems( + // The row shape's OWN variant, not `AllItems`: the two + // carry different element overheads (+15 vs +3), and a + // secondary row is a `ReferenceWithSumItem`. Describing it + // as an item under-charged every row by 12 bytes — and + // `added_bytes` is the one dimension a storage-fee + // reservation must never come in under. + estimated_layer_sizes: EstimatedLayerSizes::AllReferencesWithSumItem( secondary_key_size, row_value_size, None, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index bb85bda02..f2c5ebdc5 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -760,13 +760,22 @@ impl GroveDb { .map_err(Error::MerkError) ); let (count, sum) = element.count_sum_value_or_default(); - Ok(value_hash.map( - |value_hash| crate::operations::indexed_tree::IndexedEntryState { - count, - sum, - value_hash, - }, - )) + // A present entry with no reachable node hash is corruption, not + // absence. Collapsing the two would hand the mirror `None` and + // delete a live row (or, on the old side, skip the delete of a row + // that moved). Fail loudly instead. + let value_hash = cost_return_on_error_no_add!( + cost, + value_hash.ok_or(Error::CorruptedCodeExecution( + "an indexed primary entry exists but its node is unreachable from the \ + committed root", + )) + ); + Ok(Some(crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, + })) .wrap_with_cost(cost) } @@ -967,12 +976,19 @@ impl GroveDb { .map_err(Error::MerkError) ); let (count, sum) = element.count_sum_value_or_default(); - value_hash.map(|value_hash| { - crate::operations::indexed_tree::IndexedEntryState { - count, - sum, - value_hash, - } + // Present-but-unreachable is corruption, not + // absence: collapsing them would delete a live row. + let value_hash = cost_return_on_error_no_add!( + cost, + value_hash.ok_or(Error::CorruptedCodeExecution( + "a rewritten indexed entry exists but its node is unreachable \ + from the committed root", + )) + ); + Some(crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, }) } } @@ -1103,12 +1119,20 @@ impl GroveDb { .map_err(Error::MerkError) ); let (count, sum) = old_element.count_sum_value_or_default(); - old_value_hash.map(|value_hash| { - crate::operations::indexed_tree::IndexedEntryState { - count, - sum, - value_hash, - } + // `Element::get` above already established the entry + // exists, so a missing node hash is corruption. Treating it + // as absence would skip the delete of a row that moved. + let value_hash = cost_return_on_error_no_add!( + cost, + old_value_hash.ok_or(Error::CorruptedCodeExecution( + "an indexed primary entry exists but its node is unreachable from the \ + committed root", + )) + ); + Some(crate::operations::indexed_tree::IndexedEntryState { + count, + sum, + value_hash, }) } else { None diff --git a/grovedb/src/operations/indexed_tree.rs b/grovedb/src/operations/indexed_tree.rs index 1385db8c6..17f6fd3f9 100644 --- a/grovedb/src/operations/indexed_tree.rs +++ b/grovedb/src/operations/indexed_tree.rs @@ -1337,7 +1337,7 @@ impl GroveDb { collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode) ); drop(secondary_merk); - resolve_axis_entries(self, path, rows, transaction, grove_version).add_cost(cost) + resolve_axis_entries(self, path, rows, tx_ref, grove_version).add_cost(cost) } /// One implementation of the `indexed__top_k_paginated` shape. @@ -1381,7 +1381,7 @@ impl GroveDb { collect_top_k_via_iterator(&secondary_merk, axis, k, descending, &decode) ); drop(secondary_merk); - return resolve_axis_entries(self, path, rows, transaction, grove_version) + return resolve_axis_entries(self, path, rows, tx_ref, grove_version) .map_ok(|entries| IndexedTopKPage { entries, skipped: 0, @@ -1477,7 +1477,7 @@ impl GroveDb { } } } - resolve_axis_entries(self, path_for_resolution, rows, transaction, grove_version) + resolve_axis_entries(self, path_for_resolution, rows, tx_ref, grove_version) .map_ok(|entries| IndexedTopKPage { entries, skipped }) .add_cost(cost) } @@ -1543,7 +1543,7 @@ impl GroveDb { drop(iter); drop(secondary_merk); - resolve_axis_entries(self, path, results, transaction, grove_version).add_cost(cost) + resolve_axis_entries(self, path, results, tx_ref, grove_version).add_cost(cost) } // ---- count axis ---- @@ -2717,7 +2717,7 @@ fn resolve_axis_entries<'b, B, T>( db: &GroveDb, indexed_path: SubtreePath<'b, B>, rows: Vec<(T, Vec)>, - transaction: TransactionArg, + transaction: &Transaction, grove_version: &GroveVersion, ) -> CostResult>, Error> where @@ -2728,10 +2728,13 @@ where return Ok(Vec::new()).wrap_with_cost(cost); } - let tx = TxRef::new(&db.db, transaction); + // The CALLER's transaction, not a fresh one. The secondary scan that + // produced `rows` already ran under it; opening a second snapshot here + // would let a commit in between pair a stale row with a newer primary + // value, or report a primary the row still names as corrupted. let primary_merk = cost_return_on_error!( &mut cost, - db.open_transactional_merk_at_path(indexed_path.clone(), tx.as_ref(), None, grove_version) + db.open_transactional_merk_at_path(indexed_path.clone(), transaction, None, grove_version) ); let mut entries = Vec::with_capacity(rows.len()); @@ -2767,7 +2770,7 @@ where db.follow_reference( absolute.as_slice().into(), true, - transaction, + Some(transaction), grove_version ) ) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index df9d1da3c..9dff08a8b 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -1928,12 +1928,22 @@ impl GroveDb { *sum, ) } - TreeFeatureType::ProvableCountedMerkNode(count) => Node::KVRefValueHashCount( - key.to_owned(), - serialized_referenced_elem, - reference_element_hash, - *count, - ), + // `ProvableCountSumTree` is an eligible count-offset + // host but commits only the COUNT into its node hash + // (`binds_sum_into_hash` is true for PCPS alone), so + // its reference rows take the count-only node — the + // same variant `emit_returned_node` picks for its + // directly-valued rows. Without this arm a reference in + // such a tree hard-errored. + TreeFeatureType::ProvableCountedMerkNode(count) + | TreeFeatureType::ProvableCountedSummedMerkNode(count, _) => { + Node::KVRefValueHashCount( + key.to_owned(), + serialized_referenced_elem, + reference_element_hash, + *count, + ) + } other => { return Err(Error::CorruptedData(format!( "count-offset proof: reference row {} carries non-count feature type \ diff --git a/grovedb/src/operations/proof/indexed_axis/target_chain.rs b/grovedb/src/operations/proof/indexed_axis/target_chain.rs index 9c8443f33..60b497717 100644 --- a/grovedb/src/operations/proof/indexed_axis/target_chain.rs +++ b/grovedb/src/operations/proof/indexed_axis/target_chain.rs @@ -240,7 +240,11 @@ mod build { let mut visited = std::collections::HashSet::new(); visited.insert(head_qualified); - for _ in 0..=MAX_REFERENCE_HOPS { + // `0..`, not `0..=`: `follow_reference` allows exactly + // MAX_REFERENCE_HOPS hops, and an inclusive bound here would let + // the prover build a chain one hop deeper than `db.get` will + // follow — a proof that succeeds where the direct read refuses. + for _ in 0..MAX_REFERENCE_HOPS { let reference_path = match current_element.underlying() { Element::Reference(reference_path, ..) | Element::ReferenceWithSumItem(reference_path, ..) => reference_path.clone(), diff --git a/grovedb/src/operations/proof/indexed_axis/verify.rs b/grovedb/src/operations/proof/indexed_axis/verify.rs index 2893655c0..71acb8011 100644 --- a/grovedb/src/operations/proof/indexed_axis/verify.rs +++ b/grovedb/src/operations/proof/indexed_axis/verify.rs @@ -715,15 +715,7 @@ fn verify_indexed_axis_paginated_inner( "indexed-axis paginated proof: secondary count-offset proof failed to verify: {e}" )) })?; - let entries = decode_axis_entries_from_count_offset_items( - axis, - &count_offset_result.returned_items, - &envelope.target_chains, - grove_version, - )?; - let (secondary_root_hash, skipped) = - (count_offset_result.root_hash, count_offset_result.skipped); - + let secondary_root_hash = count_offset_result.root_hash; let initial_root = verify_deepest_layer( &envelope.layer_proofs, path, @@ -743,6 +735,19 @@ fn verify_indexed_axis_paginated_inner( "indexed-axis paginated proof", )?; + // Chain checks first, matching the range path: both reject a relabeled + // or rebound envelope, but the chain check names the actual defect + // where a row check reports only the downstream symptom — and it avoids + // folding one target chain per row for an envelope that fails anyway. + let entries = decode_axis_entries_from_count_offset_items( + axis, + &count_offset_result.returned_items, + &envelope.target_chains, + grove_version, + )?; + let (secondary_root_hash, skipped) = + (count_offset_result.root_hash, count_offset_result.skipped); + Ok(IndexedAxisPaginatedResult { root_hash, entries, diff --git a/grovedb/src/tests/count_offset_paginated_tests.rs b/grovedb/src/tests/count_offset_paginated_tests.rs index f0602a6c3..f1b4afb35 100644 --- a/grovedb/src/tests/count_offset_paginated_tests.rs +++ b/grovedb/src/tests/count_offset_paginated_tests.rs @@ -841,6 +841,73 @@ mod tests { // `merk/src/proofs/query/count_offset/tests.rs`) covers the // verifier symmetric. + /// A reference row in a `ProvableCountSumTree` host. + /// + /// That host is eligible for count-offset pagination but commits only + /// the COUNT into its node hash, so its feature type is + /// `ProvableCountedSummedMerkNode` rather than the dual-axis one. The + /// reference post-pass matched only the count-only and dual-axis + /// variants, so a reference here hard-errored on an otherwise valid + /// query. + #[test] + fn count_offset_resolves_references_in_a_provable_count_sum_tree() { + let v = GroveVersion::latest(); + let db = make_test_grovedb(v); + db.insert( + crate::tests::common::EMPTY_PATH, + b"counts", + Element::empty_provable_count_sum_tree(), + None, + None, + v, + ) + .unwrap() + .expect("insert provable count-sum tree"); + db.insert(&[b"counts"], b"a", Element::new_sum_item(5), None, None, v) + .unwrap() + .expect("insert a"); + use crate::reference_path::ReferencePathType; + db.insert( + &[b"counts"], + b"b", + Element::new_reference(ReferencePathType::SiblingReference(b"a".to_vec())), + None, + None, + v, + ) + .unwrap() + .expect("insert reference b"); + db.insert(&[b"counts"], b"c", Element::new_sum_item(7), None, None, v) + .unwrap() + .expect("insert c"); + + let mut q = Query::new(); + q.insert_range_inclusive(b"a".to_vec()..=b"z".to_vec()); + let path_query = PathQuery::new( + vec![b"counts".to_vec()], + SizedQuery::new(q, Some(2), Some(1)), + ); + let proof = db + .prove_query(&path_query, None, v) + .unwrap() + .expect("a reference in a ProvableCountSumTree must be provable"); + let (root_hash, verified) = + GroveDb::verify_query(&proof, &path_query, v).expect("proof must verify"); + assert_eq!(root_hash, db.root_hash(None, v).unwrap().unwrap()); + + let values: Vec<(Vec, Element)> = verified + .into_iter() + .map(|(_, key, element)| (key, element.expect("value present"))) + .collect(); + assert_eq!(values.len(), 2); + assert_eq!(values[0].0, b"b".to_vec()); + assert_eq!( + values[0].1, + Element::new_sum_item(5), + "the reference row must surface its dereferenced target" + ); + } + /// `Reference` in-range entries are RESOLVED, not rejected. /// /// The count-offset short-circuit used to return before the regular diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index f8e2fc64f..12e932966 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -163,6 +163,29 @@ mod tests { .collect::>()) .collect::>() ); + // Exclusivity is the point of the name: each test breaks exactly + // one part of the row, so exactly one ROW sentinel should fire. + // Without this the sentinels could quietly overlap and the + // per-corruption tests would stop distinguishing anything. + // + // Scoped to `__cidx_*` on purpose. Damaging a row moves the + // secondary root, so the indexed element's own H1-A binding stops + // matching and `verify_grovedb` reports that too — a real and + // expected consequence, not a second row diagnosis. + let row_sentinels: Vec<_> = issues + .keys() + .filter(|k| k.iter().any(|seg| seg.starts_with(b"__cidx_"))) + .map(|k| { + k.iter() + .map(|s| String::from_utf8_lossy(s).to_string()) + .collect::>() + }) + .collect(); + assert_eq!( + row_sentinels.len(), + 1, + "expected `__cidx_{kind}__` to be the only ROW sentinel, got {row_sentinels:?}" + ); } /// Baseline: an untouched tree verifies clean, and its row really is @@ -631,6 +654,32 @@ mod tests { "bulk_append left the row stale: {issues:?}" ); + // Commitment tree. Its state root moves on insert exactly like the + // other three, and it is the one non-Merk append that is LIVE on + // mainnet, so leaving it uncovered would be the worst of the four + // to get wrong. + let db = pcit_with_child( + Element::empty_commitment_tree(10).expect("valid chunk_power"), + b"ct", + ); + db.commitment_tree_insert_raw( + [TEST_LEAF, b"cidx"].as_ref(), + b"ct", + [7u8; 32], + [8u8; 32], + [9u8; 32], + vec![0u8; 216], + None, + grove_version, + ) + .unwrap() + .expect("commitment tree insert"); + let issues = db.verify_grovedb(None, true, false, grove_version).unwrap(); + assert!( + issues.is_empty(), + "commitment_tree_insert_raw left the row stale: {issues:?}" + ); + // Dense. let db = pcit_with_child(Element::empty_dense_tree(4), b"dense"); db.dense_tree_insert( From 2cee7f4ecbacb80f1426b67e94a85029881c25da Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:36:38 +0700 Subject: [PATCH 09/18] fix: drop a dead binding left by the paginated verify reorder (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo clippy --workspace --all-features -- -D warnings` — the exact CI command — caught it; my earlier per-crate --lib runs did not. --- grovedb/src/operations/proof/indexed_axis/verify.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/grovedb/src/operations/proof/indexed_axis/verify.rs b/grovedb/src/operations/proof/indexed_axis/verify.rs index 71acb8011..452322998 100644 --- a/grovedb/src/operations/proof/indexed_axis/verify.rs +++ b/grovedb/src/operations/proof/indexed_axis/verify.rs @@ -745,8 +745,7 @@ fn verify_indexed_axis_paginated_inner( &envelope.target_chains, grove_version, )?; - let (secondary_root_hash, skipped) = - (count_offset_result.root_hash, count_offset_result.skipped); + let skipped = count_offset_result.skipped; Ok(IndexedAxisPaginatedResult { root_hash, From 8bd545431f00a2e4f56423421bee7d5ca389588f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:43:04 +0700 Subject: [PATCH 10/18] fix: gate a test-only trait import behind the feature that uses it (#814) `IndexedAxisEntrySliceExt` was imported at module scope but only used by the `unsafe-dump-load`-gated test, so a default-feature `--all-targets` build saw an unused import. Moved into the gated test body. CI's lint (`--workspace --all-features`, no `--all-targets`) did not cover this; CodeRabbit's `--tests` run did. --- grovedb/src/tests/indexed_reference_row_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index 12e932966..f0145a591 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -25,7 +25,6 @@ mod tests { use crate::{ operations::indexed_tree::make_axis_secondary_key, - query_result_type::IndexedAxisEntrySliceExt, tests::{make_test_grovedb, TEST_LEAF}, Element, GroveDb, }; @@ -1076,6 +1075,7 @@ mod tests { .indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) .unwrap() .expect("top_k"); + use crate::query_result_type::IndexedAxisEntrySliceExt; assert_eq!(before.key_pairs(), vec![(3u64, b"sub".to_vec())]); // Read the child's real root so the replacement is internally From 2c25f11cca54e9dce86d57ad7b60e456f2ae1799 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:49:40 +0700 Subject: [PATCH 11/18] test: adversarial coverage for the resolved-target chain (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An indexed-axis proof hands the verifier the primary value a row points at WITHOUT a per-row inclusion proof. That saving rests on one narrow claim — the row's committed hash is bound into the secondary root, and the chain reconstructs that hash from its own bytes, so no substitution survives. The claim was argued in comments and demonstrated only by the honest path; now it is attacked directly. Twelve tests take an honest proof, decode the envelope, change exactly one thing about a chain, re-encode, and require refusal: - the resolved primary value, and the TERMINAL a reference resolves to (the attacks the design exists to stop); - the reference head itself; - a layered commitment downgraded to `Simple`, and a tampered layered child root (the element bytes stay honest, so only the fold can catch these); - a directly-valued head promoted to `Reference` with an attacker terminal appended; - a reference head with its terminal stripped, an over-long chain, an empty chain, a chain-count mismatch; - two rows' chains SWAPPED — both chains well-formed, both values genuinely in the tree, so only the per-row binding catches it. Mutation-checked so the suite is known to be discriminating rather than incidentally green: disabling the commitment comparison in `authenticate_axis_row` fails 7 of the 12, including every value substitution across all commitment shapes. The other 5 are shape guards that fire earlier, which is the intended ordering. This is the evidence for the design choice the two competing implementations disagree on. It does not settle whether per-hop path proofs buy something else — they do attest a target's current location — but it does show the returned value is unforgeable without them. 2752 grovedb tests; `clippy --workspace --all-features -- -D warnings` passes. --- .../indexed_target_chain_tamper_tests.rs | 365 ++++++++++++++++++ grovedb/src/tests/mod.rs | 1 + 2 files changed, 366 insertions(+) create mode 100644 grovedb/src/tests/indexed_target_chain_tamper_tests.rs diff --git a/grovedb/src/tests/indexed_target_chain_tamper_tests.rs b/grovedb/src/tests/indexed_target_chain_tamper_tests.rs new file mode 100644 index 000000000..b3664436f --- /dev/null +++ b/grovedb/src/tests/indexed_target_chain_tamper_tests.rs @@ -0,0 +1,365 @@ +//! Adversarial coverage for the resolved-target chain an indexed-axis +//! proof carries. +//! +//! A chain hands the verifier the primary value a secondary row points at, +//! WITHOUT a per-row inclusion proof. The claim that makes that sound is +//! narrow and worth attacking directly: the row's own committed hash is +//! bound into the secondary root, and the chain reconstructs that hash +//! from its own bytes — so no substitution anywhere in the chain can +//! survive. +//! +//! Every test here takes an honest proof, decodes the envelope, changes +//! exactly one thing about a chain, re-encodes, and asserts verification +//! refuses. If any of these ever passes, the per-row saving is not free +//! and the design is wrong. + +#[cfg(test)] +mod tests { + use bincode::config::standard; + use grovedb_element::reference_path::ReferencePathType; + use grovedb_version::version::GroveVersion; + + use crate::{ + operations::proof::indexed_axis::{ + IndexedAxisRangeProof, IndexedTargetChain, IndexedTargetCommitment, IndexedTargetNode, + }, + tests::{make_test_grovedb, TempGroveDb, TEST_LEAF}, + Element, GroveDb, + }; + + const PATH: [&[u8]; 2] = [TEST_LEAF, b"cidx"]; + + /// A PCIT holding one item-shaped entry — the simplest chain, a single + /// directly-valued node. + fn db_with_item_entry(gv: &GroveVersion) -> TempGroveDb { + let db = make_test_grovedb(gv); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + gv, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"a", + Element::new_item(b"honest-value".to_vec()), + None, + gv, + ) + .unwrap() + .expect("entry"); + db + } + + /// A PCIT whose entry is a TREE — the layered commitment shape, which + /// folds in a child root the element bytes do not carry. + fn db_with_tree_entry(gv: &GroveVersion) -> TempGroveDb { + let db = make_test_grovedb(gv); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + gv, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"a", + Element::empty_tree(), + None, + gv, + ) + .unwrap() + .expect("tree entry"); + db.insert( + [TEST_LEAF, b"cidx", b"a"].as_ref(), + b"k", + Element::new_item(b"child".to_vec()), + None, + None, + gv, + ) + .unwrap() + .expect("child"); + db + } + + /// A PCIT whose entry is a REFERENCE — a two-node chain, head plus + /// terminal. + fn db_with_reference_entry(gv: &GroveVersion) -> TempGroveDb { + let db = make_test_grovedb(gv); + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_item(b"terminal-value".to_vec()), + None, + None, + gv, + ) + .unwrap() + .expect("terminal"); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + gv, + ) + .unwrap() + .expect("PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"a", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"target".to_vec(), + ])), + None, + gv, + ) + .unwrap() + .expect("reference entry"); + db + } + + fn honest_proof(db: &TempGroveDb, gv: &GroveVersion) -> Vec { + db.prove_indexed_count_top_k(PATH.as_ref(), 5, true, None, gv) + .unwrap() + .expect("prove") + } + + /// Decode an envelope, let the caller rewrite its chains, re-encode. + fn forge(proof: &[u8], mutate: impl FnOnce(&mut Vec)) -> Vec { + let (mut envelope, _): (IndexedAxisRangeProof, _) = + bincode::decode_from_slice(proof, standard()).expect("decode envelope"); + mutate(&mut envelope.target_chains); + bincode::encode_to_vec(&envelope, standard()).expect("re-encode") + } + + #[track_caller] + fn assert_rejected(proof: &[u8], gv: &GroveVersion, what: &str) { + let res = GroveDb::verify_indexed_count_top_k(proof, PATH.as_ref(), 5, true, gv); + assert!( + res.is_err(), + "verification accepted a proof with {what}; the chain is supposed to make that \ + impossible without a per-row inclusion proof" + ); + } + + /// Baseline: the honest proofs verify, so the refusals below are + /// caused by the tampering and not by a broken fixture. + #[test] + fn honest_chains_verify_for_every_target_shape() { + let gv = GroveVersion::latest(); + for (label, db) in [ + ("item", db_with_item_entry(gv)), + ("tree", db_with_tree_entry(gv)), + ("reference", db_with_reference_entry(gv)), + ] { + let proof = honest_proof(&db, gv); + let result = GroveDb::verify_indexed_count_top_k(&proof, PATH.as_ref(), 5, true, gv) + .unwrap_or_else(|e| panic!("{label} chain must verify: {e}")); + assert_eq!( + result.root_hash, + db.root_hash(None, gv).unwrap().unwrap(), + "{label}: verified root must equal the live grove root" + ); + } + } + + /// Substituting the resolved VALUE is the attack the whole design + /// stands on refusing. If a prover could swap this, a top-k result + /// would carry an unauthenticated value. + #[test] + fn a_substituted_primary_value_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_item_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[0].value = Element::new_item(b"attacker-value".to_vec()) + .serialize(gv) + .expect("serialize"); + }); + assert_rejected(&forged, gv, "a substituted primary value"); + } + + /// Same attack one hop out: swap the TERMINAL a reference resolves to. + #[test] + fn a_substituted_terminal_value_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_reference_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + let last = chains[0].nodes.len() - 1; + chains[0].nodes[last].value = Element::new_item(b"attacker-terminal".to_vec()) + .serialize(gv) + .expect("serialize"); + }); + assert_rejected(&forged, gv, "a substituted terminal value"); + } + + /// Rewriting the reference head changes what the row committed to. + #[test] + fn a_substituted_reference_head_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_reference_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[0].value = + Element::new_reference(ReferencePathType::SiblingReference(b"elsewhere".to_vec())) + .serialize(gv) + .expect("serialize"); + }); + assert_rejected(&forged, gv, "a substituted reference head"); + } + + /// A layered target folds in a child root the element bytes do not + /// carry. Claiming `Simple` drops that fold, so the reconstructed + /// commitment would omit the subtree entirely. + #[test] + fn downgrading_a_layered_commitment_to_simple_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_tree_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[0].commitment = IndexedTargetCommitment::Simple; + }); + assert_rejected(&forged, gv, "a layered commitment downgraded to Simple"); + } + + /// Tampering the child root itself: the element bytes stay honest, so + /// only the fold can catch this. + #[test] + fn a_tampered_layered_child_root_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_tree_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[0].commitment = IndexedTargetCommitment::Layered([0xAB; 32]); + }); + assert_rejected(&forged, gv, "a tampered layered child root"); + } + + /// Promoting a directly-valued node to `Reference` and appending a + /// terminal would let a prover choose the returned value freely, since + /// a reference commits its terminal rather than its own bytes. + #[test] + fn promoting_a_direct_head_to_a_reference_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_item_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[0].commitment = IndexedTargetCommitment::Reference; + chains[0].nodes.push(IndexedTargetNode { + value: Element::new_item(b"attacker-value".to_vec()) + .serialize(gv) + .expect("serialize"), + commitment: IndexedTargetCommitment::Simple, + }); + }); + assert_rejected(&forged, gv, "a direct head promoted to a reference"); + } + + /// A reference head with its terminal stripped. The head commits the + /// terminal's hash, so without it the commitment cannot be rebuilt — + /// and treating the head as directly-valued would be the wrong fold. + #[test] + fn a_reference_head_without_its_terminal_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_reference_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes.truncate(1); + }); + assert_rejected(&forged, gv, "a reference head with no terminal"); + } + + /// Padding a chain past head-plus-terminal. Extra entries are bound by + /// nothing, so the shape is rejected rather than silently ignored. + #[test] + fn an_over_long_chain_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_reference_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + let extra = chains[0].nodes[chains[0].nodes.len() - 1].clone(); + chains[0].nodes.push(extra); + }); + assert_rejected(&forged, gv, "an over-long chain"); + } + + /// An empty chain carries no primary at all. + #[test] + fn an_empty_chain_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_item_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes.clear(); + }); + assert_rejected(&forged, gv, "an empty chain"); + } + + /// Chains are matched to rows positionally, so a count mismatch must + /// be refused rather than zipped short. + #[test] + fn a_chain_count_mismatch_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_item_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains.clear(); + }); + assert_rejected(&forged, gv, "fewer chains than rows"); + } + + /// Two entries, two rows: swapping their chains gives each row the + /// other's value. Both chains are individually well-formed and both + /// values are genuinely in the tree, so only the per-row binding + /// catches this. + #[test] + fn swapping_two_rows_chains_is_rejected() { + let gv = GroveVersion::latest(); + let db = make_test_grovedb(gv); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + gv, + ) + .unwrap() + .expect("PCIT"); + for (key, value) in [(b"a".as_ref(), b"first".as_ref()), (b"b", b"second")] { + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + key, + Element::new_item(value.to_vec()), + None, + gv, + ) + .unwrap() + .expect("entry"); + } + let honest = honest_proof(&db, gv); + let (envelope, _): (IndexedAxisRangeProof, _) = + bincode::decode_from_slice(&honest, standard()).expect("decode"); + assert_eq!( + envelope.target_chains.len(), + 2, + "fixture must produce two rows for the swap to mean anything" + ); + + let forged = forge(&honest, |chains| chains.swap(0, 1)); + assert_rejected(&forged, gv, "two rows' chains swapped"); + } +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 64e78a76a..9929e25b1 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -61,6 +61,7 @@ mod indexed_axis_offset_proof_tests; mod indexed_axis_paginated_cost_tests; mod indexed_axis_proof_tests; mod indexed_reference_row_tests; +mod indexed_target_chain_tamper_tests; mod indexed_tree_secondary_drift_tests; mod indexed_tree_security_regression_tests; mod is_empty_tree_tests; From d18cdc127a312ddf0ae036cd1fe72be61257a33a Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:58:56 +0700 Subject: [PATCH 12/18] fix: carry the count for ProvableCountSumTree references in V1 proofs (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `ProvableCountSumTree` hashes via `node_hash_with_count` — only PCPS binds the sum in — so its references need the COUNT exactly as a `ProvableCountTree`'s do. The V1 reference dispatch matched only `ProvableCountedMerkNode`, so a reference in such a tree downgraded to the aggregateless `KVRefValueHash` and the host's node hash could not be reconstructed. The proof verified nowhere. Reproduced on develop with identical hashes, so this is pre-existing and not introduced by the indexed-tree work. It surfaced because I fixed the same defect in the count-offset dispatch last round and the ordinary path was left inconsistent with it. Mutation-checked: reverting the arm reproduces the original "V1 mismatch in lower layer hash". **V0 has the identical defect and is deliberately untouched.** V0 is shipped, consensus-frozen wire format; changing what it emits is a different kind of decision from fixing a bug, and it wants its own review rather than riding along in this PR. Nothing is lost by waiting: no valid proof exists for this shape under V0 today either, so the case is unreachable through a verifying client on both envelopes. Recorded in the new test's doc comment so the asymmetry is visible rather than implied. 2753 grovedb tests; `clippy --workspace --all-features -- -D warnings` passes. --- grovedb/src/operations/proof/generate.rs | 9 +- .../tests/provable_count_sum_tree_tests.rs | 94 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 9dff08a8b..6ddb935ba 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -2031,7 +2031,14 @@ impl GroveDb { let count_for_ref = match op { Op::Push(Node::KVValueHashFeatureType(_, _, _, ft)) | Op::PushInverted(Node::KVValueHashFeatureType(_, _, _, ft)) => match ft { - TreeFeatureType::ProvableCountedMerkNode(count) => Some(*count), + // `ProvableCountSumTree` hashes via `node_hash_with_count` + // (only PCPS binds the sum in), so its references need the + // COUNT just as a `ProvableCountTree`'s do. Without this + // arm they downgraded to the aggregateless + // `KVRefValueHash` and the host's node hash could not be + // reconstructed — the proof verified nowhere. + TreeFeatureType::ProvableCountedMerkNode(count) + | TreeFeatureType::ProvableCountedSummedMerkNode(count, _) => Some(*count), _ => None, }, _ => None, diff --git a/grovedb/src/tests/provable_count_sum_tree_tests.rs b/grovedb/src/tests/provable_count_sum_tree_tests.rs index ac862bf8c..438576c58 100644 --- a/grovedb/src/tests/provable_count_sum_tree_tests.rs +++ b/grovedb/src/tests/provable_count_sum_tree_tests.rs @@ -10,6 +10,100 @@ #[cfg(test)] mod tests { + + /// An ordinary (non-count-offset) proof over a `ProvableCountSumTree` + /// that contains a Reference. + /// + /// This host hashes via `node_hash_with_count` — only + /// `ProvableCountProvableSumTree` binds the sum in — so its references + /// need the COUNT exactly as a `ProvableCountTree`'s do. The V1 + /// reference dispatch matched only `ProvableCountedMerkNode`, so a + /// reference here downgraded to the aggregateless `KVRefValueHash` and + /// the host's node hash could not be reconstructed: the proof verified + /// nowhere. + /// + /// Pre-existing on develop, not introduced by the indexed-tree work; + /// found while fixing the same defect in the count-offset dispatch. + /// + /// **V0 has the identical defect and is deliberately left alone.** It + /// is shipped, consensus-frozen wire format, so changing what it emits + /// is a separate decision from a bug fix. The case is unreachable + /// through a verifying client either way — no valid proof exists for + /// this shape under V0 today. + #[test] + fn v1_proof_over_a_provable_count_sum_tree_resolves_references() { + use grovedb_version::version::GroveVersion; + + use crate::{ + reference_path::ReferencePathType, + tests::{make_test_grovedb, TEST_LEAF}, + Element, GroveDb, PathQuery, Query, SizedQuery, + }; + + let v = GroveVersion::latest(); + let db = make_test_grovedb(v); + db.insert( + [TEST_LEAF].as_ref(), + b"pcs", + Element::empty_provable_count_sum_tree(), + None, + None, + v, + ) + .unwrap() + .expect("provable count-sum tree"); + db.insert( + [TEST_LEAF, b"pcs"].as_ref(), + b"a", + Element::new_sum_item(5), + None, + None, + v, + ) + .unwrap() + .expect("target"); + db.insert( + [TEST_LEAF, b"pcs"].as_ref(), + b"b", + Element::new_reference(ReferencePathType::SiblingReference(b"a".to_vec())), + None, + None, + v, + ) + .unwrap() + .expect("reference"); + + let mut q = Query::new(); + q.insert_all(); + let path_query = PathQuery::new( + vec![TEST_LEAF.to_vec(), b"pcs".to_vec()], + SizedQuery::new(q, None, None), + ); + let proof = db + .prove_query(&path_query, None, v) + .unwrap() + .expect("prove"); + let (root_hash, result) = GroveDb::verify_query(&proof, &path_query, v).expect( + "a reference in a \ + ProvableCountSumTree must produce a verifiable proof", + ); + assert_eq!( + root_hash, + db.root_hash(None, v).unwrap().unwrap(), + "the verified root must equal the live grove root" + ); + assert_eq!(result.len(), 2, "both entries must come back"); + + let reference_row = result + .iter() + .find(|(_, key, _)| key == b"b") + .expect("reference row"); + assert_eq!( + reference_row.2.as_ref().expect("value present"), + &Element::new_sum_item(5), + "the reference must surface its dereferenced target" + ); + } use grovedb_merk::{ proofs::{encoding::Decoder, tree::execute, Node, Op, Query}, tree::{kv::ValueDefinedCostType, AggregateData}, From 886f1cf219f727e22bf8a390a90e4eafb2990bf8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 11:39:25 +0700 Subject: [PATCH 13/18] fix: cover PrivateDocumentStoreInsert in the indexed-row exhaustive match (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit develop's #787 added `GroveOp::PrivateDocumentStoreInsert`, which `can_mutate_indexed_secondary_row` did not cover. The match is exhaustive on purpose, so the new variant was a compile error in the PR merge commit — exactly the signal it exists to produce. The op is rewritten into `ReplaceNonMerkTreeRoot` by `preprocess_private_document_store_ops` before the level executor runs, so the arm is unreachable in the current pipeline. It answers `true`, matching what the op becomes, which keeps it correct if that preprocessing is ever reordered or removed. Co-Authored-By: Claude Opus 5 --- grovedb/src/batch/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 12a68a433..0eecfc368 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -646,7 +646,13 @@ impl GroveOp { | GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => true, + | GroveOp::DenseTreeInsert { .. } + // `preprocess_private_document_store_ops` rewrites this into + // `ReplaceNonMerkTreeRoot` before the level executor runs, so + // this arm is unreachable in the current pipeline. It answers + // the same as what the op becomes, which keeps it correct if + // that preprocessing is ever reordered or removed. + | GroveOp::PrivateDocumentStoreInsert { .. } => true, } } } From 343af8fa71f1e3313a7fc2d65e62c928e0dae6f5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 12:01:26 +0700 Subject: [PATCH 14/18] test: cover the multi-axis nested primary and two chain-shape guards (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three paths the patch left untested, all reachable and worth testing on their own merit rather than for the metric: - A nested PCPSIT primary entry. A multi-axis indexed tree folds an axes DIGEST into its commitment where a single-axis one folds a bare secondary root, so it is a distinct commitment shape; a chain that rebuilt it as single-axis would not reproduce the row's hash. - A directly-valued head carrying a terminal, and a terminal that is itself a reference. These are the two chain-shape guards the existing tamper cases did not reach — the mirrors of the head-promotion and stripped-terminal cases already covered. The two new tamper cases assert on the specific rejection message, so they prove the intended guard fired rather than any guard. Adds `assert_rejected_because` for that. Also renames `mis_targeted` to `mistargeted` in an existing case: the typos hook scans the whole file once it is touched, and flagged it. Not covered, deliberately: the `KVRefValueHashCount{,Sum}` arms of the count-offset verifier. The count-offset prover emits `KVValueHashFeatureType` for reference rows (emit.rs:598), so those arms are defensive against proofs the honest prover cannot produce. The end-to-end reference-resolution behaviour they guard is already covered by `count_offset_resolves_reference_entries_to_their_target` and its count-sum sibling. Co-Authored-By: Claude Opus 5 --- .../src/tests/indexed_reference_row_tests.rs | 77 ++++++++++++++++++- .../indexed_target_chain_tamper_tests.rs | 64 +++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index f0145a591..7579f4e55 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -517,10 +517,10 @@ mod tests { /// rather than on one message keeps the test from pinning which guard /// happens to fire first. #[test] - fn an_axis_proof_over_a_mis_targeted_row_is_rejected() { + fn an_axis_proof_over_a_mistargeted_row_is_rejected() { let grove_version = GroveVersion::latest(); let db = pcit_with_one_entry(grove_version); - // A second entry, so the mis-targeted reference points at a key + // A second entry, so the mistargeted reference points at a key // that genuinely exists — the interesting case, since a dangling // reference would fail earlier and for a duller reason. db.insert_into_count_indexed_tree( @@ -771,6 +771,79 @@ mod tests { ); } + /// A NESTED MULTI-AXIS INDEXED TREE (PCPSIT) as a primary entry. + /// + /// Where a single-axis nested tree folds one secondary root into its + /// commitment, a PCPSIT folds a DIGEST over every configured axis. + /// That is a distinct commitment shape, so it needs its own case: a + /// chain that rebuilt it as a single-axis commitment would not + /// reproduce the row's hash, and the row would fail to authenticate. + #[test] + fn a_nested_multi_axis_indexed_tree_primary_proves_and_resolves() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("outer PCIT"); + // TWO axes, so the inner element's commitment folds an axes digest + // rather than a single secondary root. + let axes: Vec<(u8, Option>)> = + vec![(IndexAxis::Count.tag(), None), (IndexAxis::Sum.tag(), None)]; + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"inner", + Element::empty_provable_count_provable_sum_indexed_tree(axes).expect("axes canonical"), + None, + grove_version, + ) + .unwrap() + .expect("inner PCPSIT"); + db.insert_into_provable_count_provable_sum_indexed_tree( + [TEST_LEAF, b"cidx", b"inner"].as_ref(), + b"leaf", + Element::new_item_with_sum_item(b"v".to_vec(), 7), + None, + grove_version, + ) + .unwrap() + .expect("inner entry"); + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!(issues.is_empty(), "nested PCPSIT reported: {issues:?}"); + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("a nested multi-axis primary must be provable"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("verify"); + assert_eq!( + result.root_hash, + db.root_hash(None, grove_version).unwrap().unwrap() + ); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].primary_key, b"inner".to_vec()); + assert!( + matches!( + entries[0].value.underlying(), + Element::ProvableCountProvableSumIndexedTree(..) + ), + "the row must resolve to the nested PCPSIT itself, got {}", + entries[0].value.type_str() + ); + } + /// A REFERENCE-SHAPED primary entry resolves through to its terminal /// on both the direct and the proved path, and the two agree. /// diff --git a/grovedb/src/tests/indexed_target_chain_tamper_tests.rs b/grovedb/src/tests/indexed_target_chain_tamper_tests.rs index b3664436f..fd07ccb66 100644 --- a/grovedb/src/tests/indexed_target_chain_tamper_tests.rs +++ b/grovedb/src/tests/indexed_target_chain_tamper_tests.rs @@ -154,6 +154,26 @@ mod tests { ); } + /// Like [`assert_rejected`], but pins WHICH refusal fired. Used where + /// several guards could plausibly reject the same forgery and the test + /// is only meaningful if the intended one is the guard that ran. + #[track_caller] + fn assert_rejected_because(proof: &[u8], gv: &GroveVersion, expected: &str, what: &str) { + let res = GroveDb::verify_indexed_count_top_k(proof, PATH.as_ref(), 5, true, gv); + let err = match res { + Err(e) => e.to_string(), + Ok(_) => panic!( + "verification accepted a proof with {what}; the chain is supposed to make that \ + impossible without a per-row inclusion proof" + ), + }; + assert!( + err.contains(expected), + "{what} was rejected, but by the wrong guard: expected a message containing \ + {expected:?}, got {err:?}" + ); + } + /// Baseline: the honest proofs verify, so the refusals below are /// caused by the tampering and not by a broken fixture. #[test] @@ -282,6 +302,50 @@ mod tests { assert_rejected(&forged, gv, "a reference head with no terminal"); } + /// The mirror of [`promoting_a_direct_head_to_a_reference_is_rejected`]: + /// a terminal appended to a head left directly-valued. Only a reference + /// resolves onward, so the extra entry must be refused rather than + /// quietly taken as the resolved value. + #[test] + fn a_direct_head_carrying_a_terminal_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_item_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes.push(IndexedTargetNode { + value: Element::new_item(b"attacker-value".to_vec()) + .serialize(gv) + .expect("serialize"), + commitment: IndexedTargetCommitment::Simple, + }); + }); + assert_rejected_because( + &forged, + gv, + "a directly-valued head carries a terminal", + "a directly-valued head carrying a terminal", + ); + } + + /// A terminal relabelled as a reference. A chain must END at a directly + /// valued element — a reference terminal would commit a further hop + /// that nothing in the proof binds. + #[test] + fn a_terminal_that_is_itself_a_reference_is_rejected() { + let gv = GroveVersion::latest(); + let db = db_with_reference_entry(gv); + let honest = honest_proof(&db, gv); + let forged = forge(&honest, |chains| { + chains[0].nodes[1].commitment = IndexedTargetCommitment::Reference; + }); + assert_rejected_because( + &forged, + gv, + "the terminal entry is itself a reference", + "a terminal that is itself a reference", + ); + } + /// Padding a chain past head-plus-terminal. Extra entries are bound by /// nothing, so the shape is rejected rather than silently ignored. #[test] From 547a00267da6857f3465969b1bf61b47cd190db1 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 12:05:07 +0700 Subject: [PATCH 15/18] fix: guard unconsumed deferred_axes, not just deferred_secondary (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `deferred_secondary` and `deferred_axes` are set mutually exclusively by the same code path — single-axis (PCIT/PSIT) sets the former, PCPSIT the latter — and both are consumed by the same loop. The end-of-walk guard checked only `deferred_secondary`, so a walk that reached the root with per-axis state still staged returned Ok(()) instead of failing. That is the identical corruption the existing check catches, undetected for PCPSIT alone. Kept as a separate check with its own message so a report says WHICH half was stranded. The new test mirrors the single-axis one and is mutation-checked: disabling the guard makes it fail. Reported by CodeRabbit. Co-Authored-By: Claude Opus 5 --- grovedb/src/lib.rs | 13 +++++ grovedb/src/tests/coverage_lib_paths_tests.rs | 56 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index accf51792..ed95bf4fb 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1491,6 +1491,19 @@ impl GroveDb { )) .wrap_with_cost(cost); } + // The multi-axis half of the same invariant. `deferred_secondary` + // and `deferred_axes` are set mutually exclusively — single-axis + // (PCIT/PSIT) sets the former, PCPSIT the latter — so checking + // only one leaves the identical corruption undetected for the + // other. Kept as a separate check with its own message so a + // report says WHICH half was stranded. + if deferred_axes.is_some() { + return Err(Error::CorruptedCodeExecution( + "deferred per-axis secondary state was set but never consumed (loop reached \ + the root before updating the PCPSIT element above its primary)", + )) + .wrap_with_cost(cost); + } Ok(()).wrap_with_cost(cost) } diff --git a/grovedb/src/tests/coverage_lib_paths_tests.rs b/grovedb/src/tests/coverage_lib_paths_tests.rs index 4ce3dc783..387c9fb6d 100644 --- a/grovedb/src/tests/coverage_lib_paths_tests.rs +++ b/grovedb/src/tests/coverage_lib_paths_tests.rs @@ -559,6 +559,62 @@ mod tests { ); } + #[test] + fn propagate_at_root_with_unconsumed_deferred_axes_is_rejected() { + // The multi-axis mirror of the case above. A PCPSIT hands per-axis + // state down the same walk, and `deferred_secondary` / + // `deferred_axes` are set mutually exclusively — so a guard on only + // the single-axis half would let the identical corruption through + // for PCPSIT. Starting at the root leaves the state nowhere to be + // folded, which is the cheapest way to strand it. + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let result = { + let root_path: SubtreePath<[u8; 0]> = SubtreePath::empty(); + let merk = db + .open_transactional_merk_at_path( + root_path.clone(), + &tx, + Some(&batch), + grove_version, + ) + .unwrap() + .expect("open root merk"); + let mut merk_cache: HashMap< + SubtreePath<[u8; 0]>, + Merk, + > = HashMap::new(); + merk_cache.insert(root_path.clone(), merk); + db.propagate_changes_inner( + merk_cache, + root_path, + None, + Some(vec![(0u8, ZERO_HASH, Some(b"stale".to_vec()))]), + None, + &tx, + &batch, + grove_version, + ) + .unwrap() + }; + + match result.expect_err("unconsumed deferred axes must be rejected") { + Error::CorruptedCodeExecution(message) => assert!( + message.contains("deferred per-axis secondary state was set but never consumed"), + "unexpected message: {message}" + ), + other => panic!("expected CorruptedCodeExecution, got {other:?}"), + } + assert_eq!( + db.root_hash(None, grove_version).unwrap().unwrap(), + root_before + ); + } + // ----------------------------------------------------------------- // PCPSIT propagation: rebuilding the axes digest from on-disk state // ----------------------------------------------------------------- From fb28ed22f032415d381c88ada626064cfb7b009d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 21:49:56 +0700 Subject: [PATCH 16/18] fix: charge the mirror's bracketing primary reads in average-case estimates (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch mirror brackets the primary apply with a pre- and a post-state read of each touched entry (read_entry_aggregates, pre and post), each a Merk::get plus a Merk::get_value_hash on the primary node — the node's STORED hash is what the row must bind, and for tree- or reference-shaped entries it is a combined hash that cannot be recomputed from the element bytes. Those four node fetches per touched key were not charged, so the estimate was not a categorical superset of the write path on the seek_count / storage_loaded_bytes axes (in practice it stayed over because merk-open charges dominate, but by accident, not construction). Charged at the caller's per-key loop rather than inside average_case_indexed_secondary_mirror: the reads are per-KEY while that function is per-axis additive — one capture feeds every axis's rewrite — and the standalone mirror-cost coverage tests pin that additivity. Worst-case is untouched on purpose: its indexed gap is broader and already documented as a KNOWN GAP (WorstCaseLayerInformation cannot even identify an indexed primary). Also adds the spec §8 write-amplification fixtures: a value-only update (same count, same sum, different bytes) on PCIT and on a three-axis PCPSIT, each asserting the estimate does not come in under actual on seeks, loaded bytes, added bytes, combined written bytes, and hash calls. These are the estimated-vs-actual cases most tempted to assume "aggregates unchanged ⇒ no secondary write". Write bytes are asserted as added+replaced combined because the estimator models the row rewrite as delete+insert while the real apply replaces in place — the split differs by construction, the total must not. Co-Authored-By: Claude Fable 5 --- .../estimated_costs/average_case_costs.rs | 273 +++++++++++++++++- .../src/estimated_costs/average_case_costs.rs | 2 +- 2 files changed, 273 insertions(+), 2 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 1ef3fa5fa..aa63ca788 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -12,7 +12,8 @@ use grovedb_costs::{ }; #[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::average_case_costs::{ - add_average_case_merk_has_value, average_case_merk_propagate, EstimatedLayerInformation, + add_average_case_get_merk_node, add_average_case_merk_has_value, average_case_merk_propagate, + EstimatedLayerInformation, }; use grovedb_merk::{ element::tree_type::ElementTreeTypeExtensions, tree::AggregateData, tree_type::TreeType, @@ -888,9 +889,44 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // and the indexed arm of `average_case_cost` was unreachable. if layer_element_estimates.tree_type.is_indexed_primary() { let axes = indexed_axes_for_tree_type(layer_element_estimates.tree_type); + // The mirror brackets the primary apply with a pre- and a + // post-state read of each touched entry + // (`read_entry_aggregates`, pre and post), each a `Merk::get` + // plus a `Merk::get_value_hash` on the primary node — the + // node's STORED hash is what the row must bind, and for tree- + // or reference-shaped entries it is a combined hash that + // cannot be recomputed from the element bytes. Four node + // fetches per touched key, charged at the uncached bound (the + // post-apply pair usually hits the in-memory tree, so this + // leans over rather than under). Charged HERE and not inside + // `average_case_indexed_secondary_mirror` because the reads + // are per-key while that function is per-axis additive — one + // capture feeds every axis's rewrite. + let primary_key_width = GroveDb::average_case_layer_key_size( + &layer_element_estimates.estimated_layer_sizes, + ); + let primary_element_size = cost_return_on_error_no_add!( + cost, + layer_element_estimates + .estimated_layer_sizes + .value_with_feature_and_flags_size(grove_version) + .map_err(Error::MerkError) + ); // Once per mutated key, not once per level: the mirror rewrites // every captured key's row on every axis. for _ in 0..mirrored_key_count { + for _ in 0..4 { + cost_return_on_error_no_add!( + cost, + add_average_case_get_merk_node( + &mut cost, + primary_key_width, + primary_element_size, + layer_element_estimates.tree_type.inner_node_type(), + ) + .map_err(Error::MerkError) + ); + } cost_return_on_error!( &mut cost, GroveDb::average_case_indexed_secondary_mirror( @@ -2469,6 +2505,241 @@ mod tests { ); } + /// The spec's intentional write amplification, measured: a VALUE-ONLY + /// update — same count, same sum, different bytes — still rewrites the + /// canonical row, because the row binds the primary node's commitment + /// and that moved. This is the case the estimate is most tempted to + /// skip ("aggregates unchanged ⇒ no secondary write"), so it gets its + /// own estimated-vs-actual fixture rather than riding on the insert + /// ones above. + /// + /// `storage_loaded_bytes` is asserted here and not in the insert tests: + /// an update is where the mirror's bracketing primary reads (pre- and + /// post-state, each a node get plus a value-hash get) actually hit + /// existing nodes, so this is the fixture that would catch those reads + /// going uncharged. Write bytes are asserted as added+replaced + /// combined: the estimator models the row rewrite as delete+insert + /// (added), the real apply as an in-place replace (replaced), so the + /// per-dimension split differs by construction while the total must + /// not come in under. + #[test] + fn test_batch_indexed_value_only_update_average_case_cost_is_not_under_actual() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + db.insert( + EMPTY_PATH, + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + Some(&tx), + grove_version, + ) + .unwrap() + .expect("create pcit"); + db.insert_into_count_indexed_tree( + [b"cidx".as_ref()].as_ref(), + b"k1", + Element::new_item(b"v1".to_vec()), + Some(&tx), + grove_version, + ) + .unwrap() + .expect("seed entry"); + + // Same key, same count contribution, different bytes. + let ops = vec![QualifiedGroveDbOp::replace_op( + vec![b"cidx".to_vec()], + b"k1".to_vec(), + Element::new_item(b"v2".to_vec()), + )]; + + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(1), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }, + ); + paths.insert( + KeyInfoPath(vec![KeyInfo::KnownKey(b"cidx".to_vec())]), + EstimatedLayerInformation { + tree_type: TreeType::ProvableCountIndexedTree, + estimated_layer_count: ApproximateElements(1), + estimated_layer_sizes: AllItems(2, 2, None), + }, + ); + + let est = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + ops.clone(), + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("estimate"); + + let actual = db + .apply_batch(ops, None, Some(&tx), grove_version) + .cost_as_result() + .expect("apply value-only update"); + + assert!( + est.seek_count >= actual.seek_count, + "estimated seeks {} must not be under actual {}", + est.seek_count, + actual.seek_count + ); + assert!( + est.storage_loaded_bytes >= actual.storage_loaded_bytes, + "estimated storage_loaded_bytes {} must not be under actual {}", + est.storage_loaded_bytes, + actual.storage_loaded_bytes + ); + assert!( + est.storage_cost.added_bytes >= actual.storage_cost.added_bytes, + "estimated added_bytes {} must not be under actual {}", + est.storage_cost.added_bytes, + actual.storage_cost.added_bytes + ); + let est_written = + est.storage_cost.added_bytes as u64 + est.storage_cost.replaced_bytes as u64; + let actual_written = + actual.storage_cost.added_bytes as u64 + actual.storage_cost.replaced_bytes as u64; + assert!( + est_written >= actual_written, + "estimated written bytes {est_written} (added+replaced) must not be under actual \ + {actual_written}" + ); + assert!( + est.hash_node_calls >= actual.hash_node_calls, + "estimated hash_node_calls {} must not be under actual {}", + est.hash_node_calls, + actual.hash_node_calls + ); + } + + /// The multi-axis form of the value-only fixture above: a PCPSIT + /// rewrites the row on EVERY configured axis when the entry's + /// commitment moves, so the amplification is per-axis and the estimate + /// must scale with it. Same-sum, same-count, different bytes — sort + /// keys stay put on all three axes and only the bound commitment moves. + #[test] + fn test_batch_pcpsit_value_only_update_average_case_cost_is_not_under_actual() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + db.insert( + EMPTY_PATH, + b"idx", + Element::empty_provable_count_provable_sum_indexed_tree(vec![ + (0u8, None), + (1u8, None), + (2u8, None), + ]) + .expect("canonical axes"), + None, + Some(&tx), + grove_version, + ) + .unwrap() + .expect("create pcpsit"); + db.insert_into_provable_count_provable_sum_indexed_tree( + [b"idx".as_ref()].as_ref(), + b"k1", + Element::new_item_with_sum_item(b"v1".to_vec(), 777), + Some(&tx), + grove_version, + ) + .unwrap() + .expect("seed entry"); + + let ops = vec![QualifiedGroveDbOp::replace_op( + vec![b"idx".to_vec()], + b"k1".to_vec(), + Element::new_item_with_sum_item(b"v2".to_vec(), 777), + )]; + + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(1), + estimated_layer_sizes: AllSubtrees(3, NoSumTrees, None), + }, + ); + paths.insert( + KeyInfoPath(vec![KeyInfo::KnownKey(b"idx".to_vec())]), + EstimatedLayerInformation { + tree_type: TreeType::ProvableCountProvableSumIndexedTree, + estimated_layer_count: ApproximateElements(1), + estimated_layer_sizes: + grovedb_merk::estimated_costs::average_case_costs::EstimatedLayerSizes::AllItemsWithSumItem(2, 2, None), + }, + ); + + let est = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + ops.clone(), + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| { + Ok((NoStorageRemoval, NoStorageRemoval)) + }, + grove_version, + ) + .cost_as_result() + .expect("estimate"); + + let actual = db + .apply_batch(ops, None, Some(&tx), grove_version) + .cost_as_result() + .expect("apply value-only update"); + + assert!( + est.seek_count >= actual.seek_count, + "estimated seeks {} must not be under actual {}", + est.seek_count, + actual.seek_count + ); + assert!( + est.storage_loaded_bytes >= actual.storage_loaded_bytes, + "estimated storage_loaded_bytes {} must not be under actual {}", + est.storage_loaded_bytes, + actual.storage_loaded_bytes + ); + assert!( + est.storage_cost.added_bytes >= actual.storage_cost.added_bytes, + "estimated added_bytes {} must not be under actual {}", + est.storage_cost.added_bytes, + actual.storage_cost.added_bytes + ); + let est_written = + est.storage_cost.added_bytes as u64 + est.storage_cost.replaced_bytes as u64; + let actual_written = + actual.storage_cost.added_bytes as u64 + actual.storage_cost.replaced_bytes as u64; + assert!( + est_written >= actual_written, + "estimated written bytes {est_written} (added+replaced) must not be under actual \ + {actual_written}" + ); + assert!( + est.hash_node_calls >= actual.hash_node_calls, + "estimated hash_node_calls {} must not be under actual {}", + est.hash_node_calls, + actual.hash_node_calls + ); + } + /// The mirror charge must scale with the number of keys a batch mutates /// in an indexed primary, not be a flat per-level constant. /// diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 6146c3e8a..1afcaf954 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -438,7 +438,7 @@ impl GroveDb { /// Add average case for deletion into Merk /// Largest average key size described by a layer, used to derive a /// secondary index layer's key sizes from its primary's. - fn average_case_layer_key_size(sizes: &EstimatedLayerSizes) -> u32 { + pub(crate) fn average_case_layer_key_size(sizes: &EstimatedLayerSizes) -> u32 { match sizes { EstimatedLayerSizes::AllSubtrees(k, ..) | EstimatedLayerSizes::AllItems(k, ..) From 1624cf1c1ea18e4b815c74c4420a7e21ed28c024 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 21:50:25 +0700 Subject: [PATCH 17/18] test: RefreshReference swallow case, direct-vs-batch append equality, wrapper boundary (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review follow-ups, each pinning a case the suite asserted only adjacently: - A batch RefreshReference on a reference-shaped primary whose aggregates do NOT move — the "old_entry == new_entry swallow" the issue's §3 names. The refresh re-binds the primary's combined hash to the terminal's new value while (count, sum) stays put, so an aggregate-only mirror comparison would strand a stale row. RefreshReference reaches the mirror through its own op arm, so the Replace-based value-only tests did not cover it. Also asserts the intermediate state: an external terminal update alone must NOT stale the row — that locality is the point of the immediate-binding rule. - Each non-Merk append (MMR, bulk, commitment, dense) produces the IDENTICAL grove through the direct API and the batch op. The two entry points are separate implementations of the same mutation — the direct APIs refresh the row inside the propagation walk, the batch path through the mirror — so root-hash equality is the cheapest guard that they stay in sync. - A NonCounted-wrapped child is REJECTED by an indexed primary, on both write doors. This pins a boundary rather than a behaviour: direct and proved reads build their returned value differently, so a wrapper that could live in a primary would need its own read-equivalence coverage (a divergence of exactly this shape exists in the competing #816). No such coverage is needed BECAUSE the merk layer refuses wrappers in Provable* count trees; if that guard is ever relaxed, this test fails and says what to add. Co-Authored-By: Claude Fable 5 --- .../src/tests/indexed_reference_row_tests.rs | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) diff --git a/grovedb/src/tests/indexed_reference_row_tests.rs b/grovedb/src/tests/indexed_reference_row_tests.rs index 7579f4e55..eba558c54 100644 --- a/grovedb/src/tests/indexed_reference_row_tests.rs +++ b/grovedb/src/tests/indexed_reference_row_tests.rs @@ -1283,4 +1283,371 @@ mod tests { "ordinary references reported: {issues:?}" ); } + + /// A `NonCounted`-wrapped child must be REJECTED by an indexed + /// primary, on both write doors. + /// + /// This pins a boundary rather than a behaviour: direct and proved + /// reads build their returned value differently (the direct read hands + /// back the fetched element, the proved read decodes the chain head's + /// bytes), so a wrapper that could live in a primary would need its + /// own read-equivalence case — either path could strip it and every + /// unwrapped test would still pass. No such case is needed BECAUSE the + /// merk layer refuses wrappers in Provable* count trees (the count is + /// committed cryptographically, so an uncounted child has no + /// representable contribution). If that guard is ever relaxed, this + /// test fails and the read-equivalence coverage must be added. + #[test] + fn a_non_counted_wrapped_child_is_rejected_by_an_indexed_primary() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + let wrapped = Element::new_non_counted(Element::new_item(b"wrapped-value".to_vec())) + .expect("a NonCounted item is a valid wrapper"); + + // Dedicated API door. + let direct = db + .insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"w", + wrapped.clone(), + None, + grove_version, + ) + .unwrap(); + assert!( + direct.is_err(), + "the dedicated indexed insert must refuse a NonCounted child" + ); + + // Batch door. + let batch = db + .apply_batch( + vec![crate::batch::QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec()], + b"w".to_vec(), + wrapped, + )], + None, + None, + grove_version, + ) + .unwrap(); + assert!( + batch.is_err(), + "the batch path must refuse a NonCounted child in an indexed primary" + ); + + // Neither refusal may leave partial state behind. + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!(issues.is_empty(), "rejections left state: {issues:?}"); + } + + /// A batch `RefreshReference` on a reference-shaped primary whose + /// aggregates do NOT move — the exact "old_entry == new_entry swallow" + /// the issue's §3 calls out by name. + /// + /// The refresh re-binds the primary node's combined hash to the + /// terminal's new value, so `(count, sum)` is identical on both sides + /// of the op and only the committed value hash moves. A mirror that + /// compared aggregates alone would skip the row rewrite and strand a + /// stale binding; the value-only tests above cover that comparison for + /// `Replace`, but `RefreshReference` reaches the mirror through its own + /// op arm, so it needs its own case. + #[test] + fn a_batch_refresh_reference_with_unchanged_aggregates_refreshes_the_row() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_item(b"first-terminal".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("terminal"); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("PCIT"); + let reference_path = + ReferencePathType::UpstreamRootHeightReference(1, vec![b"target".to_vec()]); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + b"ref", + Element::new_reference(reference_path.clone()), + None, + grove_version, + ) + .unwrap() + .expect("reference-shaped primary"); + + // Move the TERMINAL first, without touching the primary. The row + // binds the IMMEDIATE node's stored hash, which has not moved, so + // the local indexed invariant must still hold — that locality is + // the point of the immediate-binding rule. (References are excluded + // from this check: the primary's stored combined hash is stale + // against its terminal by construction until the refresh lands, + // which is the caller's contract, not row corruption.) + db.insert( + [TEST_LEAF].as_ref(), + b"target", + Element::new_item(b"second-terminal".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("update terminal"); + let issues = db.verify_grovedb(None, false, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "an external terminal update must not stale the row: {issues:?}" + ); + + // Now the op under test: RefreshReference through the batch path. + db.apply_batch( + vec![crate::batch::QualifiedGroveDbOp::refresh_reference_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec()], + b"ref".to_vec(), + reference_path, + None, + None, + false, + false, + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch refresh reference"); + + // Everything must line up again, ordinary-reference checks included. + let issues = db.verify_grovedb(None, true, true, grove_version).unwrap(); + assert!( + issues.is_empty(), + "RefreshReference with unchanged aggregates left the row stale: {issues:?}" + ); + + // And both read paths must resolve to the NEW terminal. + let expected = Element::new_item(b"second-terminal".to_vec()); + let direct = db + .indexed_count_top_k([TEST_LEAF, b"cidx"].as_ref(), 5, true, None, grove_version) + .unwrap() + .expect("direct top_k"); + assert_eq!(direct.len(), 1); + assert_eq!(direct[0].value, expected); + + let path: &[&[u8]] = &[TEST_LEAF, b"cidx"]; + let proof = db + .prove_indexed_count_top_k(path, 5, true, None, grove_version) + .unwrap() + .expect("prove"); + let result = GroveDb::verify_indexed_count_top_k(&proof, path, 5, true, grove_version) + .expect("verify"); + let entries = match &result.entries { + crate::operations::proof::indexed_axis::AxisEntries::Count(v) => v, + other => panic!("expected count entries, got {other:?}"), + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].value, expected); + } + + /// Each non-Merk append must produce the IDENTICAL grove through the + /// direct API and the batch op. + /// + /// The two entry points are separate implementations of the same + /// mutation — the direct APIs refresh the row inside the propagation + /// walk, the batch path through the mirror — so root-hash equality is + /// the cheapest guard that the two stay in sync. Each kind is its own + /// sub-case because each direct API has its own copy of the + /// write-then-propagate sequence. + #[test] + fn each_non_merk_append_matches_between_direct_and_batch_paths() { + use crate::batch::QualifiedGroveDbOp; + + let grove_version = GroveVersion::latest(); + + let pcit_with_child = |child: Element, key: &[u8]| { + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"cidx", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("create PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"cidx"].as_ref(), + key, + child, + None, + grove_version, + ) + .unwrap() + .expect("non-Merk child"); + db + }; + let check = |direct_db: crate::tests::TempGroveDb, + batch_db: crate::tests::TempGroveDb, + label: &str| { + let direct_root = direct_db.root_hash(None, grove_version).unwrap().unwrap(); + let batch_root = batch_db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!( + direct_root, batch_root, + "{label}: the direct API and the batch op must produce the identical grove" + ); + for (db, path_label) in [(&direct_db, "direct"), (&batch_db, "batch")] { + let issues = db.verify_grovedb(None, true, false, grove_version).unwrap(); + assert!(issues.is_empty(), "{label} ({path_label}): {issues:?}"); + } + }; + + // MMR. + let direct_db = pcit_with_child(Element::empty_mmr_tree(), b"mmr"); + direct_db + .mmr_tree_append( + [TEST_LEAF, b"cidx"].as_ref(), + b"mmr", + b"leaf".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("direct mmr append"); + let batch_db = pcit_with_child(Element::empty_mmr_tree(), b"mmr"); + batch_db + .apply_batch( + vec![QualifiedGroveDbOp::mmr_tree_append_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec(), b"mmr".to_vec()], + b"leaf".to_vec(), + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch mmr append"); + check(direct_db, batch_db, "mmr"); + + // Bulk-append. + let direct_db = pcit_with_child( + Element::empty_bulk_append_tree(4).expect("bulk tree"), + b"bulk", + ); + direct_db + .bulk_append( + [TEST_LEAF, b"cidx"].as_ref(), + b"bulk", + b"v".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("direct bulk append"); + let batch_db = pcit_with_child( + Element::empty_bulk_append_tree(4).expect("bulk tree"), + b"bulk", + ); + batch_db + .apply_batch( + vec![QualifiedGroveDbOp::bulk_append_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec(), b"bulk".to_vec()], + b"v".to_vec(), + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch bulk append"); + check(direct_db, batch_db, "bulk"); + + // Commitment tree. + let direct_db = pcit_with_child( + Element::empty_commitment_tree(10).expect("valid chunk_power"), + b"ct", + ); + direct_db + .commitment_tree_insert_raw( + [TEST_LEAF, b"cidx"].as_ref(), + b"ct", + [7u8; 32], + [8u8; 32], + [9u8; 32], + vec![0u8; 216], + None, + grove_version, + ) + .unwrap() + .expect("direct commitment insert"); + let batch_db = pcit_with_child( + Element::empty_commitment_tree(10).expect("valid chunk_power"), + b"ct", + ); + batch_db + .apply_batch( + vec![QualifiedGroveDbOp::commitment_tree_insert_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec(), b"ct".to_vec()], + [7u8; 32], + [8u8; 32], + [9u8; 32], + vec![0u8; 216], + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch commitment insert"); + check(direct_db, batch_db, "commitment"); + + // Dense. + let direct_db = pcit_with_child(Element::empty_dense_tree(4), b"dense"); + direct_db + .dense_tree_insert( + [TEST_LEAF, b"cidx"].as_ref(), + b"dense", + b"v".to_vec(), + None, + grove_version, + ) + .unwrap() + .expect("direct dense insert"); + let batch_db = pcit_with_child(Element::empty_dense_tree(4), b"dense"); + batch_db + .apply_batch( + vec![QualifiedGroveDbOp::dense_tree_insert_op( + vec![TEST_LEAF.to_vec(), b"cidx".to_vec(), b"dense".to_vec()], + b"v".to_vec(), + )], + None, + None, + grove_version, + ) + .unwrap() + .expect("batch dense insert"); + check(direct_db, batch_db, "dense"); + } } From da39c2ac8d1c4ba1140472e8b218e855f945f465 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 21 Aug 2026 22:05:37 +0700 Subject: [PATCH 18/18] docs: give the book's verify examples the real signatures (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three verify_indexed_count_* examples in the count-indexed-tree chapter predate the final API and dropped arguments — top_k's example omitted `descending` and `grove_version`, and both query examples omitted `expected_limit` and `grove_version`. Copying either would not compile. Now byte-matched to the shipped signatures, with the limit bound positionally the same way the prover was called. Co-Authored-By: Claude Fable 5 --- docs/book/src/count-indexed-tree.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/book/src/count-indexed-tree.md b/docs/book/src/count-indexed-tree.md index 0993bc0d0..fdd3cb140 100644 --- a/docs/book/src/count-indexed-tree.md +++ b/docs/book/src/count-indexed-tree.md @@ -487,7 +487,13 @@ let entries: Vec> = db let proof_bytes = db .prove_indexed_count_top_k(path, k, /* descending: */ true, transaction, grove_version)? .expect("prove"); -let result = GroveDb::verify_indexed_count_top_k(&proof_bytes, &path, k)?; +let result = GroveDb::verify_indexed_count_top_k( + &proof_bytes, + path, + k, + /* descending: */ true, + grove_version, +)?; // result.entries: AxisEntries::Count(Vec>) // result.root_hash: [u8; 32] ``` @@ -562,7 +568,8 @@ let proof_bytes = db .expect("prove"); // Verify with the SAME query (positional binding): -let result = GroveDb::verify_indexed_count_query(&proof_bytes, &path, q)?; +let result = + GroveDb::verify_indexed_count_query(&proof_bytes, path, q, Some(limit), grove_version)?; ``` `prove_indexed_count_top_k` is just a thin wrapper around @@ -584,7 +591,7 @@ let mut q = MerkQuery::new(); q.insert_range(a.to_be_bytes().to_vec()..=b.to_be_bytes().to_vec()); let proof = db.prove_indexed_count_query(path, q.clone(), None, tx, grove_version)?; -let result = GroveDb::verify_indexed_count_query(&proof, &path, q)?; +let result = GroveDb::verify_indexed_count_query(&proof, path, q, None, grove_version)?; let count = result.entries.len(); let root_hash = result.root_hash;