diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index 53242f16a..2f43ac135 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -491,6 +491,30 @@ fn terminal_keys_is_legacy_until_v4() { ); } +#[test] +fn delete_internal_on_transaction_is_legacy_until_v4() { + // Reusing the already-open parent Merk for non-empty child tree deletes + // (issue #686) activates at GROVE_V4; v1-v3 are live in production and + // must keep the legacy reopen labeled with the child's tree type. + for v in [&GROVE_V1, &GROVE_V2, &GROVE_V3] { + assert_eq!( + v.grovedb_versions + .operations + .delete + .delete_internal_on_transaction, + 0 + ); + } + assert_eq!( + GROVE_V4 + .grovedb_versions + .operations + .delete + .delete_internal_on_transaction, + 1 + ); +} + #[test] fn v1_replication_all_zero() { let rep = &GROVE_V1.grovedb_versions.replication; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index aceb109f8..6d885d959 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -236,7 +236,12 @@ pub const GROVE_V4: GroveVersion = GroveVersion { delete_if_empty_tree: 0, delete_if_empty_tree_with_sectional_storage_function: 0, delete_operation_for_delete_internal: 0, - delete_internal_on_transaction: 0, + // v1: reuse the already-open parent Merk when deleting a + // non-empty child tree instead of reopening the parent layer + // with the child's tree type (issue #686). v0 (GROVE_V1..V3) + // keeps the legacy reopen byte-for-byte for replay + // compatibility. + delete_internal_on_transaction: 1, delete_internal_without_transaction: 0, average_case_delete_operation_for_delete: 0, worst_case_delete_operation_for_delete: 0, diff --git a/grovedb/src/operations/delete/delete_internal_on_transaction/mod.rs b/grovedb/src/operations/delete/delete_internal_on_transaction/mod.rs new file mode 100644 index 000000000..866fef358 --- /dev/null +++ b/grovedb/src/operations/delete/delete_internal_on_transaction/mod.rs @@ -0,0 +1,111 @@ +//! `delete_internal_on_transaction` — versioned dispatch. +//! +//! Deletes a single element (a value or a subtree, optionally non-empty) +//! from an already-committed state on a transaction. Every `GroveDb::delete` +//! family call routes through here. +//! +//! When the deleted element is a **non-empty child tree**, the parent layer +//! has to be mutated and its new link hash propagated upward. How the parent +//! Merk is obtained for that mutation is **consensus-critical** and +//! version-gated (issue #686): +//! +//! * **[v0]** — legacy behaviour, frozen for `GROVE_V1`..`GROVE_V3` (all live +//! in production). The parent Merk is reopened labeled with the **deleted +//! child's** tree type. The label is wrong: for the six Provable* types +//! (whose link hash embeds the aggregate via `hash_for_link`) a mismatched +//! parent/child pairing either commits a wrong link hash into the +//! grandparent (Provable* parent, plain child) or panics in +//! `hash_for_link` (plain parent, Provable* child). Any such delete that +//! already executed on a live chain committed the resulting hash into a +//! consensus root, and the reopen also has its own cost profile, so the +//! released path is kept bug-for-bug for replay compatibility. +//! * **[v1]** — `GROVE_V4`+. The already-open parent Merk is reused; it +//! carries the parent's true tree type, so delete propagation hashes and +//! aggregates with the parent type, and the redundant reopen (an extra +//! storage-context open plus `open_layered_with_root_key`) disappears. +//! The branch also propagates through the full indexed-aware walk +//! (`propagate_changes_with_transaction`, like the operation's other +//! branches) instead of the legacy batch propagation, so a delete nested +//! inside an indexed-tree primary's child subtree re-mirrors the +//! primary's canonical secondary row instead of erroring (PSIT / PCPSIT) +//! or desyncing the count index (PCIT). +//! +//! The two implementations differ ONLY in that non-empty-child-tree branch; +//! everything else is identical. See [v0] / [v1]. +//! +//! [v0]: self::v0 +//! [v1]: self::v1 + +mod v0; +mod v1; + +use grovedb_costs::{ + storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::Error as MerkError; +use grovedb_path::SubtreePath; +use grovedb_storage::StorageBatch; +use grovedb_version::version::GroveVersion; + +use super::DeleteOptions; +use crate::{Error, GroveDb, Transaction}; + +impl GroveDb { + /// Delete an element on a transaction, clearing a non-empty subtree's + /// storage when the options allow it, and propagate the parent layer's + /// new link hash upward. + /// + /// Version dispatch (consensus-critical) — see the module documentation. + pub(crate) fn delete_internal_on_transaction>( + &self, + path: SubtreePath, + key: &[u8], + options: &DeleteOptions, + transaction: &Transaction, + sectioned_removal: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + MerkError, + >, + batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + match grove_version + .grovedb_versions + .operations + .delete + .delete_internal_on_transaction + { + 0 => self.delete_internal_on_transaction_v0( + path, + key, + options, + transaction, + sectioned_removal, + batch, + grove_version, + ), + 1 => self.delete_internal_on_transaction_v1( + path, + key, + options, + transaction, + sectioned_removal, + batch, + grove_version, + ), + version => Err( + grovedb_version::error::GroveVersionError::UnknownVersionMismatch { + method: "delete_internal_on_transaction".to_string(), + known_versions: vec![0, 1], + received: version, + } + .into(), + ) + .wrap_with_cost(OperationCost::default()), + } + } +} diff --git a/grovedb/src/operations/delete/delete_internal_on_transaction/v0.rs b/grovedb/src/operations/delete/delete_internal_on_transaction/v0.rs new file mode 100644 index 000000000..e454df21b --- /dev/null +++ b/grovedb/src/operations/delete/delete_internal_on_transaction/v0.rs @@ -0,0 +1,425 @@ +//! `delete_internal_on_transaction` — **v0** (legacy, `GROVE_V1`..`GROVE_V3`). +//! +//! CONSENSUS-CRITICAL. When deleting a **non-empty child tree**, the parent +//! Merk is reopened via `Merk::open_layered_with_root_key` labeled with the +//! **deleted child's** tree type instead of the parent's (issue #686). The +//! label is wrong, and for the six Provable* types — whose link hash embeds +//! the aggregate via `hash_for_link` — a mismatched parent/child pairing +//! either: +//! +//! * commits a wrong link hash into the grandparent (Provable* parent, +//! plain child; `verify_grovedb` reports the mismatch), or +//! * panics in `hash_for_link` with "feature_type is inconsistent with its +//! tree_type" (plain parent, Provable* child, parent left non-empty). +//! +//! For non-Provable mismatches the resulting state is identical to +//! [`super::v1`] (aggregates derive from node feature types and +//! `hash_for_link` falls through to a plain node hash), but the reopen still +//! incurs its own seek/load costs, which are fee-relevant. +//! +//! `GROVE_V1`..`GROVE_V3` are live in production: any such delete that +//! already executed committed the resulting (possibly corrupt) link hash +//! into a consensus root, so replay must reproduce this path bug-for-bug — +//! including the wrong label, the extra reopen costs, and the panic. +//! +//! v0 differs from [`super::v1`] ONLY in that non-empty-child-tree branch: +//! there the already-open parent Merk (correctly labeled) is reused and the +//! reopen disappears. Everything else is identical. See the module docs in +//! [`super`][`mod@super`] for the version rationale. + +use std::collections::HashMap; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, + storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::{ + element::{ + costs::ElementCostExtensions, delete::ElementDeleteFromStorageExtensions, + tree_type::ElementTreeTypeExtensions, + }, + Error as MerkError, Merk, +}; +use grovedb_path::SubtreePath; +use grovedb_storage::{ + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, + Storage, StorageBatch, +}; +use grovedb_version::version::GroveVersion; + +use super::super::DeleteOptions; +use crate::{Element, Error, GroveDb, Transaction}; + +impl GroveDb { + /// Legacy delete on a transaction — reopens the parent layer labeled + /// with the deleted child's tree type when deleting a non-empty child + /// tree. Frozen for `GROVE_V1`..`GROVE_V3`; see the module docs. + pub(crate) fn delete_internal_on_transaction_v0>( + &self, + path: SubtreePath, + key: &[u8], + options: &DeleteOptions, + transaction: &Transaction, + sectioned_removal: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + MerkError, + >, + batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw(path.clone(), key.as_ref(), Some(transaction), grove_version) + ); + let mut subtree_to_delete_from = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + transaction, + Some(batch), + grove_version + ) + ); + // A generic delete cannot mirror the removed child's ordering value + // out of an indexed primary's secondary index. Reject before any + // mutation. + cost_return_on_error_no_add!( + cost, + crate::operations::indexed_tree::reject_generic_write_into_indexed_primary( + subtree_to_delete_from.tree_type, + "delete", + ) + ); + + let parent_tree_type = subtree_to_delete_from.tree_type; + if let Some(tree_type) = element.tree_type() { + let subtree_merk_path = path.derive_owned_with_child(key); + let subtree_merk_path_ref = SubtreePath::from(&subtree_merk_path); + + // Tree types that store data in the data namespace as non-Merk + // entries (CommitmentTree, MmrTree, BulkAppendTree, DenseTree) + // have an always-empty Merk but may have data. We cannot iterate + // their storage with find_subtrees because the entries are not + // valid Element serializations. + let non_merk_data = element.uses_non_merk_data_storage(); + + let subtree_of_tree_we_are_deleting = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + subtree_merk_path_ref.clone(), + transaction, + Some(batch), + grove_version, + ) + ); + + // For non-Merk data trees the raw_iter check would see non-Merk + // keys and wrongly report the tree as non-empty. Use the + // element's own count instead. + let is_empty = if non_merk_data { + element.non_merk_entry_count().unwrap_or(0) == 0 + } else { + subtree_of_tree_we_are_deleting + .is_empty_tree() + .unwrap_add_cost(&mut cost) + }; + + if !options.allow_deleting_non_empty_trees && !is_empty { + return if options.deleting_non_empty_trees_returns_error { + Err(Error::DeletingNonEmptyTree( + "trying to do a delete operation for a non empty tree, but options not \ + allowing this", + )) + .wrap_with_cost(cost) + } else { + Ok(false).wrap_with_cost(cost) + }; + } + + // Indexed-tree primaries own one or more secondary storage + // namespaces (the axis-ordered secondary indexes) at prefixes + // derived from the primary's prefix via S2-B + // (`Blake3(primary_prefix ‖ axis_tag)`) — one per axis: PCIT + // has Count, PSIT has Sum, and PCPSIT has up to all three + // (Count/Sum/Avg). `find_subtrees` only walks the primary's + // namespace, so without this explicit clear the secondary's + // storage would be orphaned: future inserts under the same + // path could collide with stale entries (the derived prefix + // is identical for a recreated tree at the same path), and + // the secondary index would be unreachable but would still + // consume disk. Run unconditionally on every indexed-tree + // primary delete — including the `is_empty` branch below, + // since a stale (drifted) secondary can co-exist with an + // empty primary (e.g. a bug that mirrors deletions into the + // primary but fails to mirror into the secondary would leave + // orphans here; defend against it by clearing the namespace + // at delete time). We sweep all three axis tags + // unconditionally rather than decoding the axes TLV, matching + // the nested-subtree sweep inside the `find_subtrees` loop + // below: clearing an unused axis prefix is idempotent on an + // empty namespace, so the redundancy is intentional + // defense-in-depth. The per-prefix cleanup inside that loop + // also clears these same namespaces for the target prefix, + // but both clears are idempotent so the redundancy is fine. + if tree_type.is_indexed_primary() { + let primary_prefix = RocksDbStorage::build_prefix(subtree_merk_path_ref.clone()) + .unwrap_add_cost(&mut cost); + for axis in [ + grovedb_element::indexed::IndexAxis::Count, + grovedb_element::indexed::IndexAxis::Sum, + grovedb_element::indexed::IndexAxis::Avg, + ] { + let secondary_prefix = + RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) + .unwrap_add_cost(&mut cost); + let mut secondary_storage = self + .db + .get_transactional_storage_context_by_subtree_prefix( + secondary_prefix, + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + secondary_storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup indexed-tree secondary (axis {:?}) from \ + storage: {e}", + axis + )) + }) + ); + } + } + + if !is_empty { + if non_merk_data { + // Non-Merk data trees: clear the subtree storage directly. + // These trees never contain child subtrees so we only need + // to clear the one storage context. + let mut storage = self + .db + .get_transactional_storage_context( + subtree_merk_path_ref.clone(), + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup non-merk tree data from storage: {e}", + )) + }) + ); + } else { + let subtrees_paths = cost_return_on_error!( + &mut cost, + self.find_subtrees( + &subtree_merk_path_ref, + Some(transaction), + grove_version + ) + ); + for subtree_path in subtrees_paths { + let p: SubtreePath<_> = subtree_path.as_slice().into(); + let mut storage = self + .db + .get_transactional_storage_context(p.clone(), Some(batch), transaction) + .unwrap_add_cost(&mut cost); + + cost_return_on_error!( + &mut cost, + storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup tree from storage: {e}", + )) + }) + ); + + // NESTED INDEXED-TREE SECONDARY CLEANUP. + // find_subtrees enumerates every nested subtree + // under the deletion target, but only the + // primary's storage namespace is reachable via + // the path-prefix walk. Any nested indexed-tree + // primary inside the deleted subtree has its own + // per-axis secondary namespaces at + // Blake3(its_prefix ‖ axis_tag) — one for PCIT + // (count), one for PSIT (sum), up to three for + // PCPSIT — that find_subtrees cannot see; clear + // them all too. + // + // Clearing the secondary prefix is idempotent on + // non-indexed subtrees (their secondary namespace + // is empty), so we sweep all three axis tags + // unconditionally rather than decoding each + // subtree's root element to check the tree type — + // cheaper and removes a class of missed-decoding + // bugs. + let primary_prefix = + RocksDbStorage::build_prefix(p).unwrap_add_cost(&mut cost); + for axis in [ + grovedb_element::indexed::IndexAxis::Count, + grovedb_element::indexed::IndexAxis::Sum, + grovedb_element::indexed::IndexAxis::Avg, + ] { + let secondary_prefix = + RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) + .unwrap_add_cost(&mut cost); + let mut secondary_storage = self + .db + .get_transactional_storage_context_by_subtree_prefix( + secondary_prefix, + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + secondary_storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup nested indexed-tree secondary \ + (axis {:?}) in delete: {e}", + axis + )) + }) + ); + } + } + } + // Legacy behavior (frozen — see the module docs): reopen the + // parent layer labeled with the DELETED CHILD's tree type + // before deleting the tree element. + let storage = self + .db + .get_transactional_storage_context(path.clone(), Some(batch), transaction) + .unwrap_add_cost(&mut cost); + + // The merk reopened here is the PARENT merk (at `path`), but + // the historical code labels it with the DELETED CHILD's + // tree type. For a PrivateDocumentStore child that label + // would trip the merk-level "no ops on a PDS Merk" + // chokepoint (the delete below applies to the parent), so + // use the parent's actual tree type for PDS deletions. + // Existing types keep the historical label byte-for-byte to + // avoid any behavior change on released paths. (PDS is + // v4-gated and GROVE_V4 dispatches to v1, so this carve-out + // is unreachable here in practice — kept for exactness.) + let reopen_tree_type = + if matches!(tree_type, grovedb_merk::TreeType::PrivateDocumentStore(_)) { + subtree_to_delete_from.tree_type + } else { + tree_type + }; + let mut merk_to_delete_tree_from = cost_return_on_error!( + &mut cost, + Merk::open_layered_with_root_key( + storage, + subtree_to_delete_from.root_key(), + reopen_tree_type, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|e| { + Error::CorruptedData(format!( + "cannot open a subtree with given root key: {e}" + )) + }) + ); + // We are deleting a tree, a tree uses 3 bytes + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut merk_to_delete_tree_from, + key, + Some(options.as_merk_options()), + true, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap< + SubtreePath, + Merk, + > = HashMap::default(); + merk_cache.insert(path.clone(), merk_to_delete_tree_from); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_batch_transaction( + batch, + merk_cache, + &path, + transaction, + grove_version, + ) + ); + } else { + // We are deleting a tree, a tree uses 3 bytes + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut subtree_to_delete_from, + key, + Some(options.as_merk_options()), + true, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap< + SubtreePath, + Merk, + > = HashMap::default(); + merk_cache.insert(path.clone(), subtree_to_delete_from); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + transaction, + batch, + grove_version + ) + ); + } + } else { + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut subtree_to_delete_from, + key, + Some(options.as_merk_options()), + false, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap, Merk> = + HashMap::default(); + merk_cache.insert(path.clone(), subtree_to_delete_from); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + transaction, + batch, + grove_version + ) + ); + } + + Ok(true).wrap_with_cost(cost) + } +} diff --git a/grovedb/src/operations/delete/delete_internal_on_transaction/v1.rs b/grovedb/src/operations/delete/delete_internal_on_transaction/v1.rs new file mode 100644 index 000000000..976b2d69d --- /dev/null +++ b/grovedb/src/operations/delete/delete_internal_on_transaction/v1.rs @@ -0,0 +1,393 @@ +//! `delete_internal_on_transaction` — **v1** (`GROVE_V4`+). +//! +//! Fixes issue #686: when deleting a **non-empty child tree**, the +//! already-open parent Merk (`subtree_to_delete_from`) is reused for the +//! tree-element delete and the upward propagation. It carries the parent's +//! true tree type, so `hash_for_link` and aggregate propagation run with the +//! parent type — where [`super::v0`] reopened the parent labeled with the +//! deleted **child's** tree type, corrupting the grandparent link hash for +//! Provable* parents and panicking in `hash_for_link` for Provable* children +//! under plain parents. The redundant reopen (an extra storage-context open +//! plus `Merk::open_layered_with_root_key`) disappears with it, which also +//! changes the operation's cost profile — hence the version gate. +//! +//! The same branch also propagates through +//! `propagate_changes_with_transaction` (the full indexed-aware walk used +//! by the other two branches) instead of the legacy +//! `propagate_changes_with_batch_transaction`, whose basic parent update +//! cannot climb through an indexed-tree primary: deleting a non-empty tree +//! inside one of a primary's child subtrees erred with "can only propagate +//! on tree items" for PSIT / PCPSIT and silently desynced the count index +//! for PCIT. +//! +//! v1 differs from [`super::v0`] ONLY in that non-empty-child-tree branch. +//! Everything else is identical. See the module docs in +//! [`super`][`mod@super`] for the version rationale. + +use std::collections::HashMap; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, + storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::{ + element::{delete::ElementDeleteFromStorageExtensions, tree_type::ElementTreeTypeExtensions}, + Error as MerkError, Merk, +}; +use grovedb_path::SubtreePath; +use grovedb_storage::{ + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, + Storage, StorageBatch, +}; +use grovedb_version::version::GroveVersion; + +use super::super::DeleteOptions; +use crate::{Element, Error, GroveDb, Transaction}; + +impl GroveDb { + /// Delete on a transaction, reusing the already-open (correctly labeled) + /// parent Merk when deleting a non-empty child tree. `GROVE_V4`+; see + /// the module docs. + pub(crate) fn delete_internal_on_transaction_v1>( + &self, + path: SubtreePath, + key: &[u8], + options: &DeleteOptions, + transaction: &Transaction, + sectioned_removal: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + MerkError, + >, + batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw(path.clone(), key.as_ref(), Some(transaction), grove_version) + ); + let mut subtree_to_delete_from = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + transaction, + Some(batch), + grove_version + ) + ); + // A generic delete cannot mirror the removed child's ordering value + // out of an indexed primary's secondary index. Reject before any + // mutation. + cost_return_on_error_no_add!( + cost, + crate::operations::indexed_tree::reject_generic_write_into_indexed_primary( + subtree_to_delete_from.tree_type, + "delete", + ) + ); + + let parent_tree_type = subtree_to_delete_from.tree_type; + if let Some(tree_type) = element.tree_type() { + let subtree_merk_path = path.derive_owned_with_child(key); + let subtree_merk_path_ref = SubtreePath::from(&subtree_merk_path); + + // Tree types that store data in the data namespace as non-Merk + // entries (CommitmentTree, MmrTree, BulkAppendTree, DenseTree) + // have an always-empty Merk but may have data. We cannot iterate + // their storage with find_subtrees because the entries are not + // valid Element serializations. + let non_merk_data = element.uses_non_merk_data_storage(); + + let subtree_of_tree_we_are_deleting = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + subtree_merk_path_ref.clone(), + transaction, + Some(batch), + grove_version, + ) + ); + + // For non-Merk data trees the raw_iter check would see non-Merk + // keys and wrongly report the tree as non-empty. Use the + // element's own count instead. + let is_empty = if non_merk_data { + element.non_merk_entry_count().unwrap_or(0) == 0 + } else { + subtree_of_tree_we_are_deleting + .is_empty_tree() + .unwrap_add_cost(&mut cost) + }; + + if !options.allow_deleting_non_empty_trees && !is_empty { + return if options.deleting_non_empty_trees_returns_error { + Err(Error::DeletingNonEmptyTree( + "trying to do a delete operation for a non empty tree, but options not \ + allowing this", + )) + .wrap_with_cost(cost) + } else { + Ok(false).wrap_with_cost(cost) + }; + } + + // Indexed-tree primaries own one or more secondary storage + // namespaces (the axis-ordered secondary indexes) at prefixes + // derived from the primary's prefix via S2-B + // (`Blake3(primary_prefix ‖ axis_tag)`) — one per axis: PCIT + // has Count, PSIT has Sum, and PCPSIT has up to all three + // (Count/Sum/Avg). `find_subtrees` only walks the primary's + // namespace, so without this explicit clear the secondary's + // storage would be orphaned: future inserts under the same + // path could collide with stale entries (the derived prefix + // is identical for a recreated tree at the same path), and + // the secondary index would be unreachable but would still + // consume disk. Run unconditionally on every indexed-tree + // primary delete — including the `is_empty` branch below, + // since a stale (drifted) secondary can co-exist with an + // empty primary (e.g. a bug that mirrors deletions into the + // primary but fails to mirror into the secondary would leave + // orphans here; defend against it by clearing the namespace + // at delete time). We sweep all three axis tags + // unconditionally rather than decoding the axes TLV, matching + // the nested-subtree sweep inside the `find_subtrees` loop + // below: clearing an unused axis prefix is idempotent on an + // empty namespace, so the redundancy is intentional + // defense-in-depth. The per-prefix cleanup inside that loop + // also clears these same namespaces for the target prefix, + // but both clears are idempotent so the redundancy is fine. + if tree_type.is_indexed_primary() { + let primary_prefix = RocksDbStorage::build_prefix(subtree_merk_path_ref.clone()) + .unwrap_add_cost(&mut cost); + for axis in [ + grovedb_element::indexed::IndexAxis::Count, + grovedb_element::indexed::IndexAxis::Sum, + grovedb_element::indexed::IndexAxis::Avg, + ] { + let secondary_prefix = + RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) + .unwrap_add_cost(&mut cost); + let mut secondary_storage = self + .db + .get_transactional_storage_context_by_subtree_prefix( + secondary_prefix, + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + secondary_storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup indexed-tree secondary (axis {:?}) from \ + storage: {e}", + axis + )) + }) + ); + } + } + + if !is_empty { + if non_merk_data { + // Non-Merk data trees: clear the subtree storage directly. + // These trees never contain child subtrees so we only need + // to clear the one storage context. + let mut storage = self + .db + .get_transactional_storage_context( + subtree_merk_path_ref.clone(), + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup non-merk tree data from storage: {e}", + )) + }) + ); + } else { + let subtrees_paths = cost_return_on_error!( + &mut cost, + self.find_subtrees( + &subtree_merk_path_ref, + Some(transaction), + grove_version + ) + ); + for subtree_path in subtrees_paths { + let p: SubtreePath<_> = subtree_path.as_slice().into(); + let mut storage = self + .db + .get_transactional_storage_context(p.clone(), Some(batch), transaction) + .unwrap_add_cost(&mut cost); + + cost_return_on_error!( + &mut cost, + storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup tree from storage: {e}", + )) + }) + ); + + // NESTED INDEXED-TREE SECONDARY CLEANUP. + // find_subtrees enumerates every nested subtree + // under the deletion target, but only the + // primary's storage namespace is reachable via + // the path-prefix walk. Any nested indexed-tree + // primary inside the deleted subtree has its own + // per-axis secondary namespaces at + // Blake3(its_prefix ‖ axis_tag) — one for PCIT + // (count), one for PSIT (sum), up to three for + // PCPSIT — that find_subtrees cannot see; clear + // them all too. + // + // Clearing the secondary prefix is idempotent on + // non-indexed subtrees (their secondary namespace + // is empty), so we sweep all three axis tags + // unconditionally rather than decoding each + // subtree's root element to check the tree type — + // cheaper and removes a class of missed-decoding + // bugs. + let primary_prefix = + RocksDbStorage::build_prefix(p).unwrap_add_cost(&mut cost); + for axis in [ + grovedb_element::indexed::IndexAxis::Count, + grovedb_element::indexed::IndexAxis::Sum, + grovedb_element::indexed::IndexAxis::Avg, + ] { + let secondary_prefix = + RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) + .unwrap_add_cost(&mut cost); + let mut secondary_storage = self + .db + .get_transactional_storage_context_by_subtree_prefix( + secondary_prefix, + Some(batch), + transaction, + ) + .unwrap_add_cost(&mut cost); + cost_return_on_error!( + &mut cost, + secondary_storage.clear().map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup nested indexed-tree secondary \ + (axis {:?}) in delete: {e}", + axis + )) + }) + ); + } + } + } + // Fixed behavior (issue #686): reuse the already-open parent + // Merk, which carries the parent's true tree type, so delete + // propagation hashes and aggregates with the parent type + // instead of the deleted child's. + // We are deleting a tree, a tree uses 3 bytes + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut subtree_to_delete_from, + key, + Some(options.as_merk_options()), + true, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap< + SubtreePath, + Merk, + > = HashMap::default(); + merk_cache.insert(path.clone(), subtree_to_delete_from); + // Propagate through the full indexed-aware walk, matching + // the empty-tree and plain-element branches below. The + // legacy batch propagation performed only the basic parent + // update, so a walk climbing through an indexed-tree + // primary (a non-empty tree deleted INSIDE one of the + // primary's child subtrees) left the primary's canonical + // secondary row stale — erroring with "can only propagate + // on tree items" for PSIT / PCPSIT elements and silently + // desyncing the count index for PCIT. + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + transaction, + batch, + grove_version + ) + ); + } else { + // We are deleting a tree, a tree uses 3 bytes + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut subtree_to_delete_from, + key, + Some(options.as_merk_options()), + true, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap< + SubtreePath, + Merk, + > = HashMap::default(); + merk_cache.insert(path.clone(), subtree_to_delete_from); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + transaction, + batch, + grove_version + ) + ); + } + } else { + cost_return_on_error_into!( + &mut cost, + Element::delete_with_sectioned_removal_bytes( + &mut subtree_to_delete_from, + key, + Some(options.as_merk_options()), + false, + parent_tree_type, + sectioned_removal, + grove_version, + ) + ); + let mut merk_cache: HashMap, Merk> = + HashMap::default(); + merk_cache.insert(path.clone(), subtree_to_delete_from); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + transaction, + batch, + grove_version + ) + ); + } + + Ok(true).wrap_with_cost(cost) + } +} diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index aaf56779e..604760ab2 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -14,6 +14,10 @@ #[cfg(feature = "estimated_costs")] mod average_case; +/// Versioned dispatch for `delete_internal_on_transaction` (the shared +/// delete path). Consensus-critical — see the module docs. +#[cfg(feature = "minimal")] +mod delete_internal_on_transaction; #[cfg(feature = "minimal")] mod delete_up_tree; #[cfg(feature = "estimated_costs")] @@ -24,7 +28,6 @@ use std::collections::{BTreeSet, HashMap}; #[cfg(feature = "minimal")] pub use delete_up_tree::DeleteUpTreeOptions; -use grovedb_costs::cost_return_on_error_into; #[cfg(feature = "minimal")] use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, @@ -32,8 +35,7 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::element::{ - costs::ElementCostExtensions, decode::ElementDecodeExtensions, - delete::ElementDeleteFromStorageExtensions, tree_type::ElementTreeTypeExtensions, + decode::ElementDecodeExtensions, tree_type::ElementTreeTypeExtensions, }; #[cfg(feature = "minimal")] use grovedb_merk::{proofs::Query, KVIterator, MaybeTree}; @@ -42,8 +44,7 @@ use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; use grovedb_path::SubtreePath; #[cfg(feature = "minimal")] use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, - Storage, StorageBatch, StorageContext, + rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch, StorageContext, }; use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; @@ -719,380 +720,6 @@ impl GroveDb { } } } - - fn delete_internal_on_transaction>( - &self, - path: SubtreePath, - key: &[u8], - options: &DeleteOptions, - transaction: &Transaction, - sectioned_removal: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - MerkError, - >, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult { - check_grovedb_v0_with_cost!( - "delete_internal_on_transaction", - grove_version - .grovedb_versions - .operations - .delete - .delete_internal_on_transaction - ); - - let mut cost = OperationCost::default(); - - let element = cost_return_on_error!( - &mut cost, - self.get_raw(path.clone(), key.as_ref(), Some(transaction), grove_version) - ); - let mut subtree_to_delete_from = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - path.clone(), - transaction, - Some(batch), - grove_version - ) - ); - // A generic delete cannot mirror the removed child's ordering value - // out of an indexed primary's secondary index. Reject before any - // mutation. - cost_return_on_error_no_add!( - cost, - crate::operations::indexed_tree::reject_generic_write_into_indexed_primary( - subtree_to_delete_from.tree_type, - "delete", - ) - ); - - let uses_sum_tree = subtree_to_delete_from.tree_type; - if let Some(tree_type) = element.tree_type() { - let subtree_merk_path = path.derive_owned_with_child(key); - let subtree_merk_path_ref = SubtreePath::from(&subtree_merk_path); - - // Tree types that store data in the data namespace as non-Merk - // entries (CommitmentTree, MmrTree, BulkAppendTree, DenseTree) - // have an always-empty Merk but may have data. We cannot iterate - // their storage with find_subtrees because the entries are not - // valid Element serializations. - let non_merk_data = element.uses_non_merk_data_storage(); - - let subtree_of_tree_we_are_deleting = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - subtree_merk_path_ref.clone(), - transaction, - Some(batch), - grove_version, - ) - ); - - // For non-Merk data trees the raw_iter check would see non-Merk - // keys and wrongly report the tree as non-empty. Use the - // element's own count instead. - let is_empty = if non_merk_data { - element.non_merk_entry_count().unwrap_or(0) == 0 - } else { - subtree_of_tree_we_are_deleting - .is_empty_tree() - .unwrap_add_cost(&mut cost) - }; - - if !options.allow_deleting_non_empty_trees && !is_empty { - return if options.deleting_non_empty_trees_returns_error { - Err(Error::DeletingNonEmptyTree( - "trying to do a delete operation for a non empty tree, but options not \ - allowing this", - )) - .wrap_with_cost(cost) - } else { - Ok(false).wrap_with_cost(cost) - }; - } - - // Indexed-tree primaries own one or more secondary storage - // namespaces (the axis-ordered secondary indexes) at prefixes - // derived from the primary's prefix via S2-B - // (`Blake3(primary_prefix ‖ axis_tag)`) — one per axis: PCIT - // has Count, PSIT has Sum, and PCPSIT has up to all three - // (Count/Sum/Avg). `find_subtrees` only walks the primary's - // namespace, so without this explicit clear the secondary's - // storage would be orphaned: future inserts under the same - // path could collide with stale entries (the derived prefix - // is identical for a recreated tree at the same path), and - // the secondary index would be unreachable but would still - // consume disk. Run unconditionally on every indexed-tree - // primary delete — including the `is_empty` branch below, - // since a stale (drifted) secondary can co-exist with an - // empty primary (e.g. a bug that mirrors deletions into the - // primary but fails to mirror into the secondary would leave - // orphans here; defend against it by clearing the namespace - // at delete time). We sweep all three axis tags - // unconditionally rather than decoding the axes TLV, matching - // the nested-subtree sweep inside the `find_subtrees` loop - // below: clearing an unused axis prefix is idempotent on an - // empty namespace, so the redundancy is intentional - // defense-in-depth. The per-prefix cleanup inside that loop - // also clears these same namespaces for the target prefix, - // but both clears are idempotent so the redundancy is fine. - if tree_type.is_indexed_primary() { - let primary_prefix = RocksDbStorage::build_prefix(subtree_merk_path_ref.clone()) - .unwrap_add_cost(&mut cost); - for axis in [ - grovedb_element::indexed::IndexAxis::Count, - grovedb_element::indexed::IndexAxis::Sum, - grovedb_element::indexed::IndexAxis::Avg, - ] { - let secondary_prefix = - RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) - .unwrap_add_cost(&mut cost); - let mut secondary_storage = self - .db - .get_transactional_storage_context_by_subtree_prefix( - secondary_prefix, - Some(batch), - transaction, - ) - .unwrap_add_cost(&mut cost); - cost_return_on_error!( - &mut cost, - secondary_storage.clear().map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup indexed-tree secondary (axis {:?}) from \ - storage: {e}", - axis - )) - }) - ); - } - } - - if !is_empty { - if non_merk_data { - // Non-Merk data trees: clear the subtree storage directly. - // These trees never contain child subtrees so we only need - // to clear the one storage context. - let mut storage = self - .db - .get_transactional_storage_context( - subtree_merk_path_ref.clone(), - Some(batch), - transaction, - ) - .unwrap_add_cost(&mut cost); - cost_return_on_error!( - &mut cost, - storage.clear().map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup non-merk tree data from storage: {e}", - )) - }) - ); - } else { - let subtrees_paths = cost_return_on_error!( - &mut cost, - self.find_subtrees( - &subtree_merk_path_ref, - Some(transaction), - grove_version - ) - ); - for subtree_path in subtrees_paths { - let p: SubtreePath<_> = subtree_path.as_slice().into(); - let mut storage = self - .db - .get_transactional_storage_context(p.clone(), Some(batch), transaction) - .unwrap_add_cost(&mut cost); - - cost_return_on_error!( - &mut cost, - storage.clear().map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup tree from storage: {e}", - )) - }) - ); - - // NESTED INDEXED-TREE SECONDARY CLEANUP. - // find_subtrees enumerates every nested subtree - // under the deletion target, but only the - // primary's storage namespace is reachable via - // the path-prefix walk. Any nested indexed-tree - // primary inside the deleted subtree has its own - // per-axis secondary namespaces at - // Blake3(its_prefix ‖ axis_tag) — one for PCIT - // (count), one for PSIT (sum), up to three for - // PCPSIT — that find_subtrees cannot see; clear - // them all too. - // - // Clearing the secondary prefix is idempotent on - // non-indexed subtrees (their secondary namespace - // is empty), so we sweep all three axis tags - // unconditionally rather than decoding each - // subtree's root element to check the tree type — - // cheaper and removes a class of missed-decoding - // bugs. - let primary_prefix = - RocksDbStorage::build_prefix(p).unwrap_add_cost(&mut cost); - for axis in [ - grovedb_element::indexed::IndexAxis::Count, - grovedb_element::indexed::IndexAxis::Sum, - grovedb_element::indexed::IndexAxis::Avg, - ] { - let secondary_prefix = - RocksDbStorage::secondary_prefix_for(&primary_prefix, axis.tag()) - .unwrap_add_cost(&mut cost); - let mut secondary_storage = self - .db - .get_transactional_storage_context_by_subtree_prefix( - secondary_prefix, - Some(batch), - transaction, - ) - .unwrap_add_cost(&mut cost); - cost_return_on_error!( - &mut cost, - secondary_storage.clear().map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup nested indexed-tree secondary \ - (axis {:?}) in delete: {e}", - axis - )) - }) - ); - } - } - } - // todo: verify why we need to open the same? merk again - let storage = self - .db - .get_transactional_storage_context(path.clone(), Some(batch), transaction) - .unwrap_add_cost(&mut cost); - - // The merk reopened here is the PARENT merk (at `path`), but - // the historical code labels it with the DELETED CHILD's - // tree type. For a PrivateDocumentStore child that label - // would trip the merk-level "no ops on a PDS Merk" - // chokepoint (the delete below applies to the parent), so - // use the parent's actual tree type for PDS deletions. - // Existing types keep the historical label byte-for-byte to - // avoid any behavior change on released paths. - let reopen_tree_type = - if matches!(tree_type, grovedb_merk::TreeType::PrivateDocumentStore(_)) { - subtree_to_delete_from.tree_type - } else { - tree_type - }; - let mut merk_to_delete_tree_from = cost_return_on_error!( - &mut cost, - Merk::open_layered_with_root_key( - storage, - subtree_to_delete_from.root_key(), - reopen_tree_type, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|e| { - Error::CorruptedData(format!( - "cannot open a subtree with given root key: {e}" - )) - }) - ); - // We are deleting a tree, a tree uses 3 bytes - cost_return_on_error_into!( - &mut cost, - Element::delete_with_sectioned_removal_bytes( - &mut merk_to_delete_tree_from, - key, - Some(options.as_merk_options()), - true, - uses_sum_tree, - sectioned_removal, - grove_version, - ) - ); - let mut merk_cache: HashMap< - SubtreePath, - Merk, - > = HashMap::default(); - merk_cache.insert(path.clone(), merk_to_delete_tree_from); - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_batch_transaction( - batch, - merk_cache, - &path, - transaction, - grove_version, - ) - ); - } else { - // We are deleting a tree, a tree uses 3 bytes - cost_return_on_error_into!( - &mut cost, - Element::delete_with_sectioned_removal_bytes( - &mut subtree_to_delete_from, - key, - Some(options.as_merk_options()), - true, - uses_sum_tree, - sectioned_removal, - grove_version, - ) - ); - let mut merk_cache: HashMap< - SubtreePath, - Merk, - > = HashMap::default(); - merk_cache.insert(path.clone(), subtree_to_delete_from); - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_transaction( - merk_cache, - path, - transaction, - batch, - grove_version - ) - ); - } - } else { - cost_return_on_error_into!( - &mut cost, - Element::delete_with_sectioned_removal_bytes( - &mut subtree_to_delete_from, - key, - Some(options.as_merk_options()), - false, - uses_sum_tree, - sectioned_removal, - grove_version, - ) - ); - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - merk_cache.insert(path.clone(), subtree_to_delete_from); - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_transaction( - merk_cache, - path, - transaction, - batch, - grove_version - ) - ); - } - - Ok(true).wrap_with_cost(cost) - } } #[cfg(feature = "minimal")] @@ -1102,7 +729,7 @@ mod tests { storage_cost::{removal::StorageRemovedBytes::BasicStorageRemoval, StorageCost}, OperationCost, }; - use grovedb_version::version::GroveVersion; + use grovedb_version::version::{v3::GROVE_V3, GroveVersion}; use pretty_assertions::assert_eq; use crate::{ @@ -1114,6 +741,478 @@ mod tests { Element, Error, }; + /// Issue #686 regression: deleting a non-empty child tree must keep + /// delete propagation on the PARENT's tree type. Under the legacy path + /// the parent Merk was reopened labeled with the child's tree type. + #[test] + fn test_non_empty_tree_delete_under_count_tree_parent_updates_count() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"parent", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful count tree insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child tree insert"); + db.insert( + [TEST_LEAF, b"parent", b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + let before = db + .get([TEST_LEAF].as_ref(), b"parent", None, grove_version) + .unwrap() + .expect("expected parent count tree"); + assert!(matches!(before, Element::CountTree(_, 1, _))); + + db.delete( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete non-empty child tree"); + + let after = db + .get([TEST_LEAF].as_ref(), b"parent", None, grove_version) + .unwrap() + .expect("expected parent count tree"); + assert!(matches!(after, Element::CountTree(_, 0, _))); + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!(issues.is_empty(), "verification issues: {:?}", issues); + } + + /// Issue #686 regression: CountSumTree parent with a SumTree child — + /// both count and sum must settle after deleting the populated child. + #[test] + fn test_non_empty_tree_delete_under_count_sum_tree_parent_updates_count_and_sum() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"parent", + Element::empty_count_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful count sum tree insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child sum tree insert"); + db.insert( + [TEST_LEAF, b"parent", b"child"].as_ref(), + b"leaf", + Element::new_sum_item(7), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child sum item insert"); + + let before = db + .get([TEST_LEAF].as_ref(), b"parent", None, grove_version) + .unwrap() + .expect("expected parent count sum tree"); + assert!(matches!(before, Element::CountSumTree(_, 1, 7, _))); + + db.delete( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete non-empty child tree"); + + let after = db + .get([TEST_LEAF].as_ref(), b"parent", None, grove_version) + .unwrap() + .expect("expected parent count sum tree"); + assert!(matches!(after, Element::CountSumTree(_, 0, 0, _))); + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!(issues.is_empty(), "verification issues: {:?}", issues); + } + + /// Issue #686, the case that actually diverges on current code: a + /// Provable* parent's link hash embeds its aggregate + /// (`hash_for_link`), so reopening the parent labeled with the child's + /// plain-tree type (legacy path) commits a wrong link hash into the + /// grandparent. Under GROVE_V4 the already-open parent Merk is reused + /// and the binding stays consistent. + #[test] + fn test_non_empty_tree_delete_under_provable_count_tree_parent_v4_keeps_binding() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"parent", + Element::empty_provable_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful provable count tree insert"); + // A sibling keeps the parent Merk non-empty after the delete so the + // link hash is actually recomputed during propagation. + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"sibling", + Element::new_item(b"s".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful sibling insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child tree insert"); + db.insert( + [TEST_LEAF, b"parent", b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + db.delete( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete non-empty child tree"); + + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!(issues.is_empty(), "verification issues: {:?}", issues); + } + + /// GROVE_V3 must keep the legacy path byte-for-byte: the same scenario + /// as `..._v4_keeps_binding` leaves a wrong link hash in the + /// grandparent, which `verify_grovedb` reports. Any such delete that + /// happened on a live v3 chain committed that hash into a consensus + /// root, so replay must reproduce it. + #[test] + fn test_non_empty_tree_delete_under_provable_count_tree_parent_v3_keeps_legacy() { + let grove_version = &GROVE_V3; + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"parent", + Element::empty_provable_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful provable count tree insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"sibling", + Element::new_item(b"s".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful sibling insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child tree insert"); + db.insert( + [TEST_LEAF, b"parent", b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + db.delete( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("legacy delete non-empty child tree"); + + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!( + !issues.is_empty(), + "legacy v3 path is expected to leave a mismatched parent link hash; if this now \ + verifies clean, the v3 behavior changed — that breaks replay compatibility" + ); + } + + /// Issue #686, panic variant: deleting a non-empty Provable* CHILD under + /// a plain parent made the legacy path reopen the parent labeled + /// ProvableCountTree; `hash_for_link` then panics on the parent's basic + /// nodes. Fixed under GROVE_V4 by reusing the correctly-labeled parent. + #[test] + fn test_non_empty_provable_child_delete_under_normal_parent_v4_no_panic() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"child", + Element::empty_provable_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful provable count child insert"); + db.insert( + [TEST_LEAF].as_ref(), + b"sibling", + Element::new_item(b"s".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful sibling insert"); + db.insert( + [TEST_LEAF, b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + db.delete( + [TEST_LEAF].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete non-empty provable child tree"); + + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!(issues.is_empty(), "verification issues: {:?}", issues); + } + + /// Pins the legacy panic on GROVE_V3 (see + /// `..._v4_no_panic`). If this stops panicking the v3 code path + /// changed, which would break replay compatibility. + #[test] + #[should_panic(expected = "ProvableCountTree::hash_for_link")] + fn test_non_empty_provable_child_delete_under_normal_parent_v3_panics() { + let grove_version = &GROVE_V3; + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"child", + Element::empty_provable_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful provable count child insert"); + db.insert( + [TEST_LEAF].as_ref(), + b"sibling", + Element::new_item(b"s".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful sibling insert"); + db.insert( + [TEST_LEAF, b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + let _ = db.delete( + [TEST_LEAF].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ); + } + + /// GROVE_V3 keeps the legacy (version 0) delete path for plain aggregate + /// parents: the delete still lands and — on current code — the count + /// settles correctly (aggregates are read from node feature types, so + /// the mislabeled reopen is benign for non-Provable parents). + #[test] + fn test_legacy_non_empty_tree_delete_keeps_version_0_path() { + let grove_version = &GROVE_V3; + let db = make_test_grovedb(grove_version); + + db.insert( + [TEST_LEAF].as_ref(), + b"parent", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful count tree insert"); + db.insert( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child tree insert"); + db.insert( + [TEST_LEAF, b"parent", b"child"].as_ref(), + b"leaf", + Element::new_item(b"value".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("successful child item insert"); + + db.delete( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("legacy delete non-empty child tree"); + + assert!(matches!( + db.get( + [TEST_LEAF, b"parent"].as_ref(), + b"child", + None, + grove_version + ) + .unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + let after = db + .get([TEST_LEAF].as_ref(), b"parent", None, grove_version) + .unwrap() + .expect("expected parent count tree"); + assert!(matches!(after, Element::CountTree(_, 0, _))); + } + #[test] fn test_empty_subtree_deletion_without_transaction() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/tests/delete_indexed_tree_tests.rs b/grovedb/src/tests/delete_indexed_tree_tests.rs index 8dc93f8b3..bbb512fb0 100644 --- a/grovedb/src/tests/delete_indexed_tree_tests.rs +++ b/grovedb/src/tests/delete_indexed_tree_tests.rs @@ -147,6 +147,89 @@ mod tests { assert!(get_result.is_err()); } + /// A generic non-empty tree delete NESTED BELOW an indexed primary must + /// re-mirror the primary's secondary row on the way up: the delete + /// changes the child subtree's root (and possibly its count), and the + /// canonical secondary row binds that node's commitment. The v1 + /// (GROVE_V4) delete path routes propagation through the full + /// indexed-aware walk; the legacy batch propagation performed only the + /// basic parent update and left the secondary stale. + #[test] + fn delete_non_empty_tree_nested_below_pcit_remirrors_secondary() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"pcit", + Element::empty_provable_count_indexed_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("create PCIT"); + db.insert_into_count_indexed_tree( + [TEST_LEAF, b"pcit"].as_ref(), + b"a", + Element::empty_provable_count_tree(), + None, + grove_version, + ) + .unwrap() + .expect("insert PCIT entry"); + // Populate the entry subtree generically (allowed: `a` is a plain + // ProvableCountTree, not a primary), including a nested non-empty + // tree whose deletion will exercise the non-empty delete branch. + db.insert( + [TEST_LEAF, b"pcit", b"a"].as_ref(), + b"sibling", + Element::new_item(vec![]), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert sibling item"); + db.insert( + [TEST_LEAF, b"pcit", b"a"].as_ref(), + b"t", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert nested tree"); + db.insert( + [TEST_LEAF, b"pcit", b"a", b"t"].as_ref(), + b"leaf", + Element::new_item(b"v".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert nested leaf"); + + // Delete the NON-EMPTY nested tree `t`. Its parent `a` is not a + // primary, so the generic-write rejection does not fire, and the + // propagation walk climbs a -> pcit (indexed primary) -> test_leaf. + db.delete( + [TEST_LEAF, b"pcit", b"a"].as_ref(), + b"t", + Some(delete_options_allow_non_empty()), + None, + grove_version, + ) + .unwrap() + .expect("delete non-empty nested tree below a PCIT entry"); + + let issues = db + .verify_grovedb(None, false, true, grove_version) + .expect("verify grovedb"); + assert!(issues.is_empty(), "verification issues: {:?}", issues); + } + #[test] fn delete_empty_pcit_succeeds_and_clears_secondary() { let grove_version = GroveVersion::latest();