From 638c839ac62f1788eb098f1509b247572ba3c383 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 13 Nov 2026 19:49:39 +0700 Subject: [PATCH 1/3] fix(drive)!: copy retained revisions in the history migration without decoding them (PV15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protocol version 15 history migration copies retained revisions without decoding their properties, batches its writes, and keeps the owner's storage flags on the migrated per-document history tree. Builds on #4652. - dpp: `DocumentV0::id_and_revision_from_bytes` reads a serialized document's id and revision from the fields every serialization format (0 to 3) writes ahead of its properties. The migration reads each retained revision's id and revision this way instead of decoding the revision with the type's current schema, and copies its bytes unchanged. - drive: the migration gathers its writes across documents, types and contracts and applies them as two ordered batches (history trees, revisions and legacy-tree deletes, then current pointers and index references) once they reach `max_history_migration_batch_operations` operations, a new `SystemLimits` field: 500 in `SYSTEM_LIMITS_V5`, `None` in the earlier tables, which never run the migration. The cap also bounds the stack a batch needs, since Merk's batch delete recurses about once per key it removes from a tree. The read-back of rewritten index references is dropped; the tests read them back. With one operation per batch the result, root hash included, is that of the per-document migration. - drive: the migrated `[2]/` history tree keeps the flags of the legacy per-document tree it replaces instead of taking the contract's. What the first protocol version 15 block writes changes relative to 5.1.0-dev.1; networks that activated it on that prerelease need a reset. Test would have caught this in CI: should_migrate_revisions_stored_before_a_nested_property_change and should_keep_the_owner_flags_on_the_migrated_history_tree ✖ before the fix, ✔ after. should_store_the_same_histories_whatever_the_batch_size compares the stored elements across batch sizes, reads every index reference back and verifies GroveDB's hashes. Co-Authored-By: Claude Opus 5.5 --- .../src/document/v0/serialize/header.rs | 289 ++++++ .../rs-dpp/src/document/v0/serialize/mod.rs | 1 + .../v1/mod.rs | 5 +- .../src/drive/document/lifecycle/tests.rs | 17 +- .../drive/document/migration/index_tests.rs | 2 +- .../src/drive/document/migration/mod.rs | 269 +++--- .../src/drive/document/migration/tests.rs | 877 ++++++++++++++++-- .../src/version/mocks/v2_test.rs | 1 + .../src/version/system_limits/mod.rs | 16 + .../src/version/system_limits/v1.rs | 1 + .../src/version/system_limits/v2.rs | 1 + .../src/version/system_limits/v3.rs | 1 + .../src/version/system_limits/v4.rs | 1 + .../src/version/system_limits/v5.rs | 4 + .../rs-platform-version/src/version/v15.rs | 2 +- 15 files changed, 1251 insertions(+), 236 deletions(-) create mode 100644 packages/rs-dpp/src/document/v0/serialize/header.rs diff --git a/packages/rs-dpp/src/document/v0/serialize/header.rs b/packages/rs-dpp/src/document/v0/serialize/header.rs new file mode 100644 index 00000000000..2ff9b212e81 --- /dev/null +++ b/packages/rs-dpp/src/document/v0/serialize/header.rs @@ -0,0 +1,289 @@ +use crate::consensus::basic::decode::DecodingError; +use crate::data_contract::document_type::accessors::DocumentTypeV0Getters; +use crate::data_contract::document_type::methods::DocumentTypeBasicMethods; +use crate::data_contract::document_type::DocumentTypeRef; +use crate::data_contract::errors::DataContractError; +use crate::document::v0::DocumentV0; +use crate::nft::TradeMode; +use crate::prelude::Revision; +use crate::ProtocolError; +use integer_encoding::VarIntReader; +use platform_value::Identifier; +use platform_version::version::FeatureVersion; +use std::io::Read; + +impl DocumentV0 { + /// Reads the id and the revision of a serialized document from the fields + /// every serialization format (0 to 3) writes ahead of the document's + /// properties, without decoding the properties. + /// + /// The fields are the format, the contract version stamp (format 3), the + /// id, the owner id, the creator id (formats 2 and 3, for a type whose + /// documents can change hands) and the revision (for a type whose documents + /// carry one). Which of them are present depends on the type's + /// configuration, which a contract update cannot change, and never on its + /// properties. Bytes stored before an update changed how the properties + /// are encoded (the member order of a nested object, an optional member + /// added ahead of others) give their id and revision here, although + /// [`DocumentV0::from_bytes`] with the updated type no longer reads them. + /// For bytes [`DocumentV0::from_bytes`] reads, this returns the id and the + /// revision it decodes. + /// + /// The history migration on the first block of protocol version 15 keys + /// every retained revision by what this returns, so what it accepts and + /// returns for bytes written in formats 0 to 3 must never change. + pub fn id_and_revision_from_bytes( + mut serialized_document: &[u8], + document_type: DocumentTypeRef, + ) -> Result<(Identifier, Option), ProtocolError> { + let format: FeatureVersion = serialized_document + .read_varint() + .map_err(|_| header_error("serialization format"))?; + match format { + 0..=2 => {} + 3 => { + let _contract_version: u64 = serialized_document + .read_varint() + .map_err(|_| header_error("contract version stamp"))?; + } + version => { + return Err(ProtocolError::UnknownVersionMismatch { + method: "DocumentV0::id_and_revision_from_bytes".to_string(), + known_versions: vec![0, 1, 2, 3], + received: version, + }) + } + } + let mut id = [0; 32]; + serialized_document + .read_exact(&mut id) + .map_err(|_| header_error("id"))?; + let mut identifier = [0; 32]; + serialized_document + .read_exact(&mut identifier) + .map_err(|_| header_error("owner id"))?; + if format >= 2 + && (document_type.trade_mode() != TradeMode::None + || document_type.documents_transferable().is_transferable()) + { + let mut has_creator_id = [0]; + serialized_document + .read_exact(&mut has_creator_id) + .map_err(|_| header_error("creator id flag"))?; + if has_creator_id[0] > 0 { + serialized_document + .read_exact(&mut identifier) + .map_err(|_| header_error("creator id"))?; + } + } + let revision = if document_type.requires_revision() { + Some( + serialized_document + .read_varint() + .map_err(|_| header_error("revision"))?, + ) + } else { + None + }; + Ok((Identifier::new(id), revision)) + } +} + +fn header_error(field: &str) -> ProtocolError { + ProtocolError::DataContractError(DataContractError::DecodingDocumentError( + DecodingError::new(format!( + "error reading the {field} from a serialized document's header" + )), + )) +} + +#[cfg(test)] +mod tests { + use crate::data_contract::config::DataContractConfig; + use crate::data_contract::document_type::methods::DocumentTypeBasicMethods; + use crate::data_contract::document_type::DocumentType; + use crate::document::serialization_traits::DocumentPlatformConversionMethodsV0; + use crate::document::v0::DocumentV0; + use crate::ProtocolError; + use platform_value::{platform_value, Identifier, Value}; + use platform_version::version::PlatformVersion; + use std::collections::BTreeMap; + + /// A type with an object property `o` holding `members` in this order, + /// configured by `settings` (merged into the type's schema). + fn document_type(settings: Value, members: Vec<(&str, Value)>) -> DocumentType { + let platform_version = PlatformVersion::latest(); + let mut schema = platform_value!({ + "type": "object", + "properties": { + "o": { + "type": "object", "position": 0, + "properties": Value::Map( + members + .into_iter() + .map(|(name, schema)| (Value::Text(name.to_string()), schema)) + .collect(), + ), + "required": ["b", "c"], "additionalProperties": false, + }, + }, + "required": ["o"], "additionalProperties": false, + }); + let Value::Map(settings) = settings else { + panic!("type settings are a map"); + }; + for (key, value) in settings { + schema + .set_value(key.as_text().expect("a setting name"), value) + .expect("the schema is a map"); + } + let config = + DataContractConfig::default_for_version(platform_version).expect("a default config"); + DocumentType::try_from_schema( + Identifier::new([2; 32]), + 1, + config.version(), + "note", + schema, + None, + &BTreeMap::new(), + &config, + false, + &mut Vec::new(), + platform_version, + ) + .expect("a valid document type") + } + + fn members(order: &[&'static str]) -> Vec<(&'static str, Value)> { + order + .iter() + .map(|name| { + let schema = match *name { + "a" => platform_value!({"type": "string", "maxLength": 20, "position": 2}), + "b" => platform_value!({"type": "string", "maxLength": 20, "position": 0}), + "c" => { + platform_value!({"type": "integer", "minimum": 0, "maximum": 10, "position": 1}) + } + other => panic!("no member {other}"), + }; + (*name, schema) + }) + .collect() + } + + fn document(revision: Option, creator_id: Option<[u8; 32]>) -> DocumentV0 { + let mut properties = BTreeMap::new(); + properties.insert("o".to_string(), platform_value!({"b": "hello", "c": 5u8})); + DocumentV0 { + id: Identifier::new([9; 32]), + owner_id: Identifier::new([7; 32]), + creator_id: creator_id.map(Identifier::new), + properties, + revision, + created_at: Some(1_000), + ..Default::default() + } + } + + /// Every type configuration that changes which header fields are + /// written: a revision or none, and a creator block or none. + fn configurations() -> Vec<(&'static str, Value)> { + vec![ + ("mutable", platform_value!({"documentsMutable": true})), + ("immutable", platform_value!({"documentsMutable": false})), + ( + "transferable", + platform_value!({"documentsMutable": false, "transferable": 1}), + ), + ( + "sold by its owner", + platform_value!({"documentsMutable": false, "transferable": 1, "tradeMode": 1}), + ), + ] + } + + fn serialize(document: &DocumentV0, format: u16, document_type: &DocumentType) -> Vec { + let document_type = document_type.as_ref(); + match format { + 0 => document.serialize_v0(document_type), + 1 => document.serialize_v1(document_type), + 2 => document.serialize_v2(document_type), + 3 => document.serialize_v3(document_type), + _ => unreachable!("formats 0 to 3"), + } + .expect("a serializable document") + } + + #[test] + fn should_read_the_id_and_revision_from_bytes_decodes_in_every_format() { + let platform_version = PlatformVersion::latest(); + for (name, settings) in configurations() { + let document_type = document_type(settings, members(&["b", "c"])); + let revision = document_type.as_ref().requires_revision().then_some(4); + for creator_id in [None, Some([3; 32])] { + for format in 0..=3 { + let bytes = serialize(&document(revision, creator_id), format, &document_type); + let decoded = + DocumentV0::from_bytes(&bytes, document_type.as_ref(), platform_version) + .expect("bytes the type reads"); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes, document_type.as_ref()) + .expect("a readable header"), + (decoded.id, decoded.revision), + "{name} type, format {format}, creator {creator_id:?}" + ); + assert_eq!(decoded.revision, revision); + } + } + } + } + + #[test] + fn should_read_the_header_of_bytes_written_before_a_nested_member_change() { + let platform_version = PlatformVersion::latest(); + for (name, settings) in configurations() { + let written = document_type(settings.clone(), members(&["b", "c"])); + let revision = written.as_ref().requires_revision().then_some(4); + for (change, order) in [ + ("reordered", &["c", "b"][..]), + ("prepended optional member", &["a", "b", "c"][..]), + ] { + let updated = document_type(settings.clone(), members(order)); + for format in 0..=3 { + let bytes = serialize(&document(revision, Some([3; 32])), format, &written); + assert!( + DocumentV0::from_bytes(&bytes, updated.as_ref(), platform_version).is_err(), + "the {change} type no longer reads the {name} document in format {format}" + ); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes, updated.as_ref()) + .expect("a readable header"), + (Identifier::new([9; 32]), revision), + "{name} type, {change}, format {format}" + ); + } + } + } + } + + #[test] + fn should_refuse_a_header_it_cannot_read() { + let document_type = document_type( + platform_value!({"documentsMutable": true}), + members(&["b", "c"]), + ); + let bytes = serialize(&document(Some(4), None), 3, &document_type); + for truncated in [&bytes[..0], &bytes[..1], &bytes[..40], &bytes[..66]] { + assert!( + DocumentV0::id_and_revision_from_bytes(truncated, document_type.as_ref()).is_err(), + "{} bytes hold no complete header", + truncated.len() + ); + } + assert!(matches!( + DocumentV0::id_and_revision_from_bytes(&[4, 0], document_type.as_ref()), + Err(ProtocolError::UnknownVersionMismatch { received: 4, .. }) + )); + } +} diff --git a/packages/rs-dpp/src/document/v0/serialize/mod.rs b/packages/rs-dpp/src/document/v0/serialize/mod.rs index e9b41b7e24c..38f176a8bb9 100644 --- a/packages/rs-dpp/src/document/v0/serialize/mod.rs +++ b/packages/rs-dpp/src/document/v0/serialize/mod.rs @@ -25,6 +25,7 @@ use crate::consensus::ConsensusError; use crate::data_contract::accessors::v0::DataContractV0Getters; use crate::data_contract::config::DataContractConfig; +mod header; mod v0; mod v1; mod v2; diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs index ce666d9eaf8..9f4280bc854 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs @@ -190,8 +190,9 @@ impl Drive { DirectQueryType::StatelessDirectQuery { in_tree_type: TreeType::NormalTree, // A tree created at PV15 carries its creator's single-epoch owned flags; - // one migrated at activation carries the contract's flags, which can be larger. - // This is an average-case figure; the delete's total estimate covers the applied read. + // one migrated at activation keeps its legacy tree's flags, the creator's + // single-epoch owned flags or none. This is an average-case figure; the + // delete's total estimate covers the applied read. query_target: QueryTargetTree( StorageFlags::approximate_size(true, None), TreeType::ProvableCountTree, diff --git a/packages/rs-drive/src/drive/document/lifecycle/tests.rs b/packages/rs-drive/src/drive/document/lifecycle/tests.rs index d3945440d4d..99f099f4744 100644 --- a/packages/rs-drive/src/drive/document/lifecycle/tests.rs +++ b/packages/rs-drive/src/drive/document/lifecycle/tests.rs @@ -3221,6 +3221,9 @@ fn should_cover_applied_delete_fees_for_migrated_history_with_multi_epoch_contra let drive = setup_drive_with_initial_state_structure(None); let owner = [7; 32]; let mut schema = json_document_to_json_value(FAMILY_HISTORY_CONTRACT).unwrap(); + // A deletable contract, so the legacy writer flags the document's history + // tree with the document's own flags. + schema["config"] = serde_json::json!({"$formatVersion": "0", "canBeDeleted": true}); let contract = DataContract::from_json(schema.clone(), false, old).unwrap(); drive .apply_contract( @@ -3300,13 +3303,13 @@ fn should_cover_applied_delete_fees_for_migrated_history_with_multi_epoch_contra ) .value .unwrap(); - let flags = StorageFlags::map_cow_some_element_flags_ref(tree.get_flags()) - .unwrap() - .unwrap(); - assert!(matches!(flags.as_ref(), StorageFlags::MultiEpochOwned(..))); - assert!( - tree.get_flags().as_ref().unwrap().len() - > StorageFlags::approximate_size(true, None) as usize + assert_eq!( + tree.get_flags(), + &StorageFlags::map_to_some_element_flags(Some(&StorageFlags::new_single_epoch( + 0, + Some(owner) + ))), + "the migrated history tree keeps its legacy tree's flags, not the contract's" ); assert_delete_estimate_covers_applied( &drive, diff --git a/packages/rs-drive/src/drive/document/migration/index_tests.rs b/packages/rs-drive/src/drive/document/migration/index_tests.rs index 32b1aa91ce0..ac6c08e4ae4 100644 --- a/packages/rs-drive/src/drive/document/migration/index_tests.rs +++ b/packages/rs-drive/src/drive/document/migration/index_tests.rs @@ -13,7 +13,7 @@ use dpp::block::block_info::BlockInfo; use dpp::block::epoch::Epoch; use dpp::data_contract::DataContractFactory; use dpp::document::document_factory::DocumentFactory; -use dpp::document::DocumentV0Setters; +use dpp::document::{Document, DocumentV0Getters, DocumentV0Setters}; use dpp::platform_value::{platform_value, Value}; use std::borrow::Cow; diff --git a/packages/rs-drive/src/drive/document/migration/mod.rs b/packages/rs-drive/src/drive/document/migration/mod.rs index 2b70d443f5b..260f585ee57 100644 --- a/packages/rs-drive/src/drive/document/migration/mod.rs +++ b/packages/rs-drive/src/drive/document/migration/mod.rs @@ -15,8 +15,7 @@ use crate::util::storage_flags::StorageFlags; use dpp::data_contract::accessors::v0::DataContractV0Getters; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dpp::data_contract::document_type::DocumentPropertyType; -use dpp::document::serialization_traits::DocumentPlatformConversionMethodsV0; -use dpp::document::{Document, DocumentV0Getters}; +use dpp::document::DocumentV0; use dpp::version::PlatformVersion; use grovedb::batch::{QualifiedGroveDbOp, SubelementsDeletionBehavior}; use grovedb::reference_path::ReferencePathType::{SiblingReference, UpstreamRootHeightReference}; @@ -40,7 +39,7 @@ pub struct DocumentHistoryMigrationStats { pub maximum_document_revisions: u64, /// Document index references inspected. pub index_entries: u64, - /// Index references successfully rewritten. + /// Index references rewritten. pub rewritten_index_entries: u64, /// Largest index-reference count retained for one document type. pub maximum_type_index_entries: u64, @@ -55,6 +54,24 @@ pub struct DocumentHistoryMigrationStats { type IndexReference = (Vec>, Vec, Element); type IndexEntries = BTreeMap, Vec>; +/// Writes gathered across documents, types and contracts and applied together, so the root is +/// recomputed once per batch rather than once per document. +#[derive(Default)] +struct PendingHistoryWrites { + /// History trees and their revisions to insert, and the legacy per-document trees to delete. + moves: Vec, + /// Current pointers written over the deleted trees' keys, and the index references that now + /// target them. A batch can not delete a key's tree and write a reference at that key, or + /// point a reference into a tree it deletes, so these are applied after `moves`. + links: Vec, +} + +impl PendingHistoryWrites { + fn len(&self) -> usize { + self.moves.len() + self.links.len() + } +} + #[cfg(test)] mod index_tests; #[cfg(test)] @@ -99,6 +116,12 @@ fn index_inventory_buffer_bound(entries: &IndexEntries) -> u64 { impl Drive { /// Moves timestamp-keyed document histories out of primary storage. /// The caller owns the transaction and must roll it back on any error. + /// + /// Retained revisions are copied unchanged. Only their id and revision are + /// read, from the fields the serialized document holds ahead of its + /// properties, so a revision is moved whatever the type's current schema + /// makes of its properties. The writes are applied in batches of about + /// `max_history_migration_batch_operations` operations. pub fn migrate_document_history_storage( &self, transaction: &Transaction, @@ -109,6 +132,14 @@ impl Drive { "document history migration requires the composite-key layout", )); } + let max_batch_operations = platform_version + .system_limits + .max_history_migration_batch_operations + .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( + "the protocol version that migrates document histories sets \ + max_history_migration_batch_operations", + )))? as usize; + let mut pending = PendingHistoryWrites::default(); let mut stats = DocumentHistoryMigrationStats::default(); let mut cursor: Option<[u8; 32]> = None; loop { @@ -183,12 +214,14 @@ impl Drive { .iter() .any(|(key, _)| key == &[DOCUMENT_HISTORY_TREE_KEY]) { - self.history_migration_batch( - vec![QualifiedGroveDbOp::insert_or_replace_op( - type_path.clone(), - vec![DOCUMENT_HISTORY_TREE_KEY], - Element::empty_tree_with_flags(contract_flags.clone()), - )], + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( + type_path.clone(), + vec![DOCUMENT_HISTORY_TREE_KEY], + Element::empty_tree_with_flags(contract_flags.clone()), + )); + self.history_migration_apply_when_full( + &mut pending, + max_batch_operations, transaction, &mut stats, platform_version, @@ -235,11 +268,17 @@ impl Drive { )?; let mut pointer = None; let mut revisions = BTreeMap::new(); - let mut copies = vec![QualifiedGroveDbOp::insert_or_replace_op( + // The history tree keeps the flags of the per-document tree it + // replaces, which the legacy writer flagged with the document's + // own flags when the contract can be deleted, and left unflagged + // otherwise. + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( history_path[..history_path.len() - 1].to_vec(), document_id.clone(), - Element::empty_provable_count_tree_with_flags(contract_flags.clone()), - )]; + Element::empty_provable_count_tree_with_flags( + element.get_flags().clone(), + ), + )); for (key, element) in &entries { if key == &[0] { pointer = Some(element.clone()); @@ -251,12 +290,18 @@ impl Drive { let Element::Item(bytes, _) = element else { return Err(corrupt("legacy history revision must be an item")); }; - let document = Document::from_bytes( + stats.revisions += 1; + stats.revision_bytes += bytes.len() as u64; + let (id, revision) = DocumentV0::id_and_revision_from_bytes( bytes, document_type.as_ref(), - platform_version, - )?; - if document.id().as_slice() != document_id { + ) + .map_err(|error| { + corrupt(format!( + "legacy history revision has no readable header: {error}" + )) + })?; + if id.as_slice() != document_id { return Err(corrupt( "history document id does not match its primary key", )); @@ -264,97 +309,70 @@ impl Drive { let time = DocumentPropertyType::decode_date_timestamp(key) .ok_or_else(|| corrupt("invalid history timestamp"))?; let mut composite = encode_u64(time); - composite.extend(encode_u64(document.revision().unwrap_or(1))); + composite.extend(encode_u64(revision.unwrap_or(1))); revisions.insert(key.clone(), composite.clone()); - copies.push(QualifiedGroveDbOp::insert_or_replace_op( + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( history_path.clone(), composite, element.clone(), )); } - let revision_entries = entries - .iter() - .filter(|(key, _)| key != &[0]) - .cloned() - .collect::>(); - Self::history_migration_count_revisions(&revision_entries, &mut stats)?; + stats.maximum_document_revisions = + stats.maximum_document_revisions.max(revisions.len() as u64); let pointer = Self::history_migration_pointer( pointer.ok_or_else(|| corrupt("history has no current pointer"))?, &document_id, &revisions, )?; - self.history_migration_batch( - copies, - transaction, - &mut stats, - platform_version, - )?; - let mut deletes = entries - .into_iter() - .map(|(key, _)| QualifiedGroveDbOp::delete_op(old_path.clone(), key)) - .collect::>(); - deletes.push(QualifiedGroveDbOp::delete_tree_op( + pending.moves.extend( + entries.into_iter().map(|(key, _)| { + QualifiedGroveDbOp::delete_op(old_path.clone(), key) + }), + ); + pending.moves.push(QualifiedGroveDbOp::delete_tree_op( primary_path.clone(), document_id.clone(), old_tree_type, SubelementsDeletionBehavior::Error, )); - self.history_migration_batch( - deletes, - transaction, - &mut stats, - platform_version, - )?; - self.history_migration_batch( - vec![QualifiedGroveDbOp::insert_or_replace_op( - primary_path.clone(), - document_id.clone(), - pointer, - )], + pending.links.push(QualifiedGroveDbOp::insert_or_replace_op( + primary_path.clone(), + document_id.clone(), + pointer, + )); + for (path, key, mut element) in + index_entries.remove(&document_id).unwrap_or_default() + { + match &mut element { + Element::Reference(reference, hops, _) + | Element::ReferenceWithSumItem(reference, hops, _, _) => { + *reference = UpstreamRootHeightReference( + 4, + vec![vec![0], document_id.clone()], + ); + *hops = Some(2); + } + // The inventory only admits references; anything + // else is corrupt state, reported like the rest. + _ => { + return Err(corrupt( + "unexpected leaf in a historical document index", + )) + } + } + pending + .links + .push(QualifiedGroveDbOp::insert_or_replace_op(path, key, element)); + stats.rewritten_index_entries += 1; + } + stats.migrated_documents += 1; + self.history_migration_apply_when_full( + &mut pending, + max_batch_operations, transaction, &mut stats, platform_version, )?; - if let Some(references) = index_entries.remove(&document_id) { - let rewrites = references - .iter() - .cloned() - .map(|(path, key, mut element)| { - match &mut element { - Element::Reference(reference, hops, _) - | Element::ReferenceWithSumItem(reference, hops, _, _) => { - *reference = UpstreamRootHeightReference( - 4, - vec![vec![0], document_id.clone()], - ); - *hops = Some(2); - } - // The inventory only admits references; anything - // else is corrupt state, reported like the rest. - _ => { - return Err(corrupt( - "unexpected leaf in a historical document index", - )) - } - } - Ok(QualifiedGroveDbOp::insert_or_replace_op(path, key, element)) - }) - .collect::, Error>>()?; - self.history_migration_batch( - rewrites, - transaction, - &mut stats, - platform_version, - )?; - self.history_migration_check_index_rewrites( - &references, - &document_id, - transaction, - &mut stats, - platform_version, - )?; - } - stats.migrated_documents += 1; } if !index_entries.is_empty() { return Err(corrupt("history index references a missing document")); @@ -362,48 +380,50 @@ impl Drive { } } } + self.history_migration_apply(&mut pending, transaction, &mut stats, platform_version)?; Ok(stats) } - fn history_migration_check_index_rewrites( + /// Applies the pending writes once they hold `max_batch_operations` operations. + fn history_migration_apply_when_full( &self, - references: &[IndexReference], - document_id: &[u8], + pending: &mut PendingHistoryWrites, + max_batch_operations: usize, transaction: &Transaction, stats: &mut DocumentHistoryMigrationStats, platform_version: &PlatformVersion, ) -> Result<(), Error> { - let mut rewritten = 0; - for (path, key, original) in references { - let mut expected = original.clone(); - match &mut expected { - Element::Reference(reference, hops, _) - | Element::ReferenceWithSumItem(reference, hops, _, _) => { - *reference = - UpstreamRootHeightReference(4, vec![vec![0], document_id.to_vec()]); - *hops = Some(2); - } - _ => continue, + if pending.len() >= max_batch_operations { + self.history_migration_apply(pending, transaction, stats, platform_version)?; + } + Ok(()) + } + + /// Applies the pending moves, then the pending links. + fn history_migration_apply( + &self, + pending: &mut PendingHistoryWrites, + transaction: &Transaction, + stats: &mut DocumentHistoryMigrationStats, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + for operations in [ + std::mem::take(&mut pending.moves), + std::mem::take(&mut pending.links), + ] { + if operations.is_empty() { + continue; } - let stored = self - .grove - .get_raw( - path.as_slice().into(), - key, + self.grove + .apply_batch( + operations, + None, Some(transaction), &platform_version.drive.grove_version, ) .value?; - if stored == expected { - rewritten += 1; - } + stats.batches += 1; } - if rewritten != references.len() { - return Err(corrupt( - "history index rewrite count differs from its inventory", - )); - } - stats.rewritten_index_entries += rewritten as u64; Ok(()) } @@ -535,23 +555,4 @@ impl Drive { } Ok(()) } - - fn history_migration_batch( - &self, - operations: Vec, - transaction: &Transaction, - stats: &mut DocumentHistoryMigrationStats, - platform_version: &PlatformVersion, - ) -> Result<(), Error> { - self.grove - .apply_batch( - operations, - None, - Some(transaction), - &platform_version.drive.grove_version, - ) - .value?; - stats.batches += 1; - Ok(()) - } } diff --git a/packages/rs-drive/src/drive/document/migration/tests.rs b/packages/rs-drive/src/drive/document/migration/tests.rs index 300dbc4554a..471c35af85c 100644 --- a/packages/rs-drive/src/drive/document/migration/tests.rs +++ b/packages/rs-drive/src/drive/document/migration/tests.rs @@ -1,10 +1,25 @@ use super::*; +use crate::drive::document::paths::document_history_path; +use crate::drive::RootTree; use crate::query::{SingleDocumentDriveQuery, SingleDocumentDriveQueryContestedStatus}; use crate::util::object_size_info::{DocumentAndContractInfo, DocumentInfo, OwnedDocumentInfo}; use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; use dpp::block::block_info::BlockInfo; -use dpp::document::DocumentV0Setters; -use dpp::tests::json_document::{json_document_to_contract, json_document_to_document}; +use dpp::data_contract::accessors::v0::DataContractV0Setters; +use dpp::data_contract::config::v0::DataContractConfigGettersV0; +use dpp::data_contract::conversion::json::DataContractJsonConversionMethodsV0; +use dpp::data_contract::validate_update::DataContractUpdateValidationMethodsV0; +use dpp::data_contract::{DataContract, DataContractFactory}; +use dpp::document::document_factory::DocumentFactory; +use dpp::document::serialization_traits::DocumentPlatformConversionMethodsV0; +use dpp::document::{Document, DocumentV0Getters, DocumentV0Setters}; +use dpp::platform_value::{platform_value, Value}; +use dpp::tests::json_document::{ + json_document_to_contract, json_document_to_document, json_document_to_json_value, +}; +use grovedb::AggregateData; +use grovedb_merk::element::reconstruct::ElementReconstructExtensions; +use std::borrow::Cow; #[test] fn should_accept_history_migration_when_only_primary_writer_changes() { @@ -164,6 +179,16 @@ fn migrate_summable_history(revision_count: u64) -> DocumentHistoryMigrationStat let old_entries = drive .history_migration_entries(&old_path, &transaction, old) .unwrap(); + let legacy_tree = drive + .grove + .get_raw( + (&old_path[..old_path.len() - 1]).into(), + document.id().as_slice(), + Some(&transaction), + &old.drive.grove_version, + ) + .value + .unwrap(); let stats = drive .migrate_document_history_storage(&transaction, new) .unwrap(); @@ -244,8 +269,13 @@ fn migrate_summable_history(revision_count: u64) -> DocumentHistoryMigrationStat .unwrap(); assert_eq!( history_tree.get_flags(), - &StorageFlags::map_to_some_element_flags(Some(&contract_flags)), - "new structural bytes inherit the contract's flags" + legacy_tree.get_flags(), + "the history tree keeps the flags of the per-document tree it replaces" + ); + assert_eq!( + legacy_tree.get_flags(), + &None, + "the legacy writer leaves a history tree unflagged when the contract can not be deleted" ); let history_root = drive .grove @@ -352,7 +382,10 @@ fn should_migrate_revisions_and_indexes_without_recovering_overwritten_revisions assert_eq!(stats.revisions, 2); assert_eq!(stats.migrated_documents, 1); assert!(stats.index_entries > 0); - assert_eq!(stats.rewritten_index_entries, stats.index_entries); + assert_eq!( + assert_index_references_rewritten(&drive, &contract, "profile", &transaction), + stats.index_entries + ); let root_migrated = drive .grove .root_hash(Some(&transaction), &new.drive.grove_version) @@ -556,92 +589,6 @@ fn should_reject_unrecognised_type_children_during_migration_inventory() { } } -#[test] -fn should_halt_migration_when_an_inventoried_index_reference_was_not_rewritten() { - let old = PlatformVersion::get(14).unwrap(); - let new = PlatformVersion::get(15).unwrap(); - let drive = setup_drive_with_initial_state_structure(Some(old)); - let contract = json_document_to_contract( - "tests/supporting_files/contract/dashpay/dashpay-contract-with-profile-history.json", - false, - old, - ) - .unwrap(); - drive - .apply_contract(&contract, BlockInfo::default(), true, None, None, old) - .unwrap(); - let document_type = contract.document_type_for_name("profile").unwrap(); - let document = json_document_to_document( - "tests/supporting_files/contract/dashpay/profile0.json", - Some([7; 32].into()), - document_type, - old, - ) - .unwrap(); - drive - .add_document_for_contract( - DocumentAndContractInfo { - owned_document_info: OwnedDocumentInfo { - document_info: DocumentInfo::DocumentRefInfo((&document, None)), - owner_id: None, - }, - contract: &contract, - document_type, - }, - true, - BlockInfo::default_with_time(1000), - true, - None, - old, - ) - .unwrap(); - let transaction = drive.grove.start_transaction(); - let type_path = contract_document_type_path_vec(contract.id().as_slice(), "profile"); - let mut stats = DocumentHistoryMigrationStats::default(); - let mut inventory = IndexEntries::new(); - for (key, _) in drive - .history_migration_entries(&type_path, &transaction, old) - .unwrap() - { - if key == [0] { - continue; - } - let mut path = type_path.clone(); - path.push(key); - drive - .history_migration_index_entries(path, &transaction, &mut stats, &mut inventory, old) - .unwrap(); - } - let references = inventory.remove(document.id().as_slice()).unwrap(); - assert!(!references.is_empty()); - let error = drive - .history_migration_check_index_rewrites( - &references, - document.id().as_slice(), - &transaction, - &mut stats, - new, - ) - .expect_err("an omitted rewrite must halt instead of being counted as rewritten"); - assert!(error - .to_string() - .contains("history index rewrite count differs from its inventory")); - assert_eq!(stats.rewritten_index_entries, 0); - drive - .migrate_document_history_storage(&transaction, new) - .unwrap(); - drive - .history_migration_check_index_rewrites( - &references, - document.id().as_slice(), - &transaction, - &mut stats, - new, - ) - .unwrap(); - assert_eq!(stats.rewritten_index_entries, references.len() as u64); -} - /// A keep-history type with a contested index could be registered before /// protocol 15. Activation loads every stored contract through the structural /// parse, which must keep reading that combination, or the migration would @@ -727,3 +674,751 @@ fn should_migrate_a_legacy_contested_keep_history_contract() { assert_eq!(stats.types, 1); assert_eq!(stats.migrated_documents, 1); } + +/// Writes `document` of type `name` as protocol 14 does, in the block at +/// `time_ms`, with `flags`. +fn add_legacy_revision( + drive: &Drive, + contract: &DataContract, + name: &str, + document: &Document, + flags: StorageFlags, + time_ms: u64, +) { + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentInfo::DocumentRefInfo(( + document, + Some(Cow::Owned(flags)), + )), + owner_id: None, + }, + contract, + document_type: contract.document_type_for_name(name).unwrap(), + }, + true, + BlockInfo::default_with_time(time_ms), + true, + None, + PlatformVersion::get(14).unwrap(), + ) + .unwrap(); +} + +/// Checks that every index reference of type `name` targets its document's +/// current pointer, two hops from the revision it resolves to, and returns +/// how many there are. +fn assert_index_references_rewritten( + drive: &Drive, + contract: &DataContract, + name: &str, + transaction: &Transaction, +) -> u64 { + let version = PlatformVersion::latest(); + let type_path = contract_document_type_path_vec(contract.id().as_slice(), name); + let mut stats = DocumentHistoryMigrationStats::default(); + let mut inventory = IndexEntries::new(); + for index in contract + .document_type_for_name(name) + .unwrap() + .index_structure() + .sub_levels() + .keys() + { + let mut path = type_path.clone(); + path.push(index.as_bytes().to_vec()); + drive + .history_migration_index_entries(path, transaction, &mut stats, &mut inventory, version) + .unwrap(); + } + for (document_id, references) in inventory { + let current_pointer = vec![vec![0], document_id]; + for (path, key, element) in references { + assert!( + matches!( + &element, + Element::Reference(UpstreamRootHeightReference(4, target), Some(2), _) + | Element::ReferenceWithSumItem( + UpstreamRootHeightReference(4, target), + Some(2), + _, + _ + ) if target == ¤t_pointer + ), + "index reference {path:?}/{key:?} was not rewritten: {element:?}" + ); + } + } + stats.index_entries +} + +/// A keep-history type `note` whose property `o` is an object holding +/// `members`, in this order, in contract version `version`. +fn nested_member_contract(members: Vec<(&str, Value)>, version: u32) -> DataContract { + let members = Value::Map( + members + .into_iter() + .map(|(name, schema)| (Value::Text(name.to_string()), schema)) + .collect(), + ); + let schema = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": true, + "canBeDeleted": false, + "properties": { + "o": { + "type": "object", "position": 0, "properties": members, + "required": ["b", "c"], "additionalProperties": false, + }, + }, + "required": ["o"], "additionalProperties": false, + }); + let mut contract = DataContractFactory::new(14) + .unwrap() + .create_with_value_config( + [7; 32].into(), + 0, + platform_value!({"note": schema}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + contract.set_version(version); + contract +} + +/// Protocol 14 accepts a contract update that reorders a nested object's +/// members or prepends an optional one, and properties are encoded in member +/// order. Revisions stored before such an update keep their bytes, so the +/// migration moves them without reading their properties. +#[test] +fn should_migrate_revisions_stored_before_a_nested_property_change() { + let old = PlatformVersion::get(14).unwrap(); + let new = PlatformVersion::latest(); + let a = platform_value!({"type": "string", "maxLength": 20, "position": 2}); + let b = platform_value!({"type": "string", "maxLength": 20, "position": 0}); + let c = platform_value!({"type": "integer", "minimum": 0, "maximum": 10, "position": 1}); + let original = nested_member_contract(vec![("b", b.clone()), ("c", c.clone())], 1); + for (change, updated) in [ + ( + "reordered members", + nested_member_contract(vec![("c", c.clone()), ("b", b.clone())], 2), + ), + ( + "prepended optional member", + nested_member_contract( + vec![("a", a.clone()), ("b", b.clone()), ("c", c.clone())], + 2, + ), + ), + ] { + assert!( + original + .validate_update(&updated, &BlockInfo::default(), old) + .unwrap() + .is_valid(), + "protocol 14 accepts the update with {change}" + ); + let drive = setup_drive_with_initial_state_structure(Some(old)); + drive + .apply_contract(&original, BlockInfo::default(), true, None, None, old) + .unwrap(); + let mut document = DocumentFactory::new(14) + .unwrap() + .create_document( + &original, + [7; 32].into(), + "note".into(), + platform_value!({"o": {"b": "hello", "c": 5}}), + ) + .unwrap(); + document.set_id([9; 32].into()); + for revision in 1..=2u64 { + document.set_revision(Some(revision)); + document.set("o", platform_value!({"b": "hello", "c": revision + 3})); + add_legacy_revision( + &drive, + &original, + "note", + &document, + StorageFlags::new_single_epoch(revision as u16, Some([7; 32])), + revision * 1000, + ); + } + drive + .apply_contract( + &updated, + BlockInfo::default_with_time(5000), + true, + None, + None, + old, + ) + .unwrap(); + + let transaction = drive.grove.start_transaction(); + let mut legacy_path = contract_document_type_path_vec(original.id().as_slice(), "note"); + legacy_path.extend([vec![0], document.id().to_vec()]); + let legacy = drive + .history_migration_entries(&legacy_path, &transaction, old) + .unwrap() + .into_iter() + .filter(|(key, _)| key != &[0]) + .collect::>(); + assert_eq!(legacy.len(), 2); + let Element::Item(stored, _) = &legacy[0].1 else { + panic!("a retained revision is an item"); + }; + assert!( + Document::from_bytes(stored, updated.document_type_for_name("note").unwrap(), new) + .is_err(), + "the fixture must hold revisions the updated type no longer reads ({change})" + ); + + let stats = drive + .migrate_document_history_storage(&transaction, new) + .unwrap_or_else(|error| panic!("migration after {change} failed: {error}")); + assert_eq!(stats.migrated_documents, 1); + let migrated = drive + .history_migration_entries( + &document_history_path(original.id().as_slice(), "note", document.id().as_slice()), + &transaction, + new, + ) + .unwrap(); + let expected = legacy + .into_iter() + .zip(1u64..) + .map(|((time, element), revision)| { + let mut key = + encode_u64(DocumentPropertyType::decode_date_timestamp(&time).unwrap()); + key.extend(encode_u64(revision)); + (key, element) + }) + .collect::>(); + assert_eq!( + migrated, expected, + "every revision keeps its bytes and flags under its time and revision ({change})" + ); + } +} + +/// The legacy writer gave each document's history tree the document's own +/// flags when the contract can be deleted. The migrated tree keeps them, so +/// erasing the document's history credits its owner, not the contract's owner. +#[test] +fn should_keep_the_owner_flags_on_the_migrated_history_tree() { + let old = PlatformVersion::get(14).unwrap(); + let new = PlatformVersion::latest(); + let contract_owner = [66; 32]; + let document_owner = [14; 32]; + // A deletable keep-history type is refused at registration since protocol + // 14, but stored contracts from before hold them, so the stored contract is + // read without validation. + let mut schema = json_document_to_json_value( + "tests/supporting_files/contract/family/family-contract-with-history.json", + ) + .unwrap(); + schema["config"] = serde_json::json!({"$formatVersion": "0", "canBeDeleted": true}); + let contract = DataContract::from_json(schema, false, old).unwrap(); + assert!(contract.config().can_be_deleted()); + let drive = setup_drive_with_initial_state_structure(Some(old)); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + Some(Cow::Owned(StorageFlags::new_single_epoch( + 0, + Some(contract_owner), + ))), + None, + old, + ) + .unwrap(); + let document_type = contract.document_type_for_name("person").unwrap(); + let mut document = json_document_to_document( + "tests/supporting_files/contract/family/person0.json", + Some(document_owner.into()), + document_type, + old, + ) + .unwrap(); + let owner_flags = StorageFlags::new_single_epoch(0, Some(document_owner)); + for revision in 1..=2u64 { + document.set_revision(Some(revision)); + add_legacy_revision( + &drive, + &contract, + "person", + &document, + owner_flags.clone(), + revision * 1000, + ); + } + let type_path = contract_document_type_path_vec(contract.id().as_slice(), "person"); + let mut primary_path = type_path.clone(); + primary_path.push(vec![0]); + let legacy_tree = drive + .grove + .get_raw( + primary_path.as_slice().into(), + document.id().as_slice(), + None, + &old.drive.grove_version, + ) + .value + .unwrap(); + assert_eq!( + legacy_tree.get_flags(), + &StorageFlags::map_to_some_element_flags(Some(&owner_flags)), + "the legacy writer flags a deletable contract's history tree with its document's flags" + ); + + let transaction = drive.grove.start_transaction(); + drive + .migrate_document_history_storage(&transaction, new) + .unwrap(); + drive + .grove + .commit_transaction(transaction) + .unwrap() + .unwrap(); + let mut history_root = type_path.clone(); + history_root.push(vec![DOCUMENT_HISTORY_TREE_KEY]); + let history_tree = drive + .grove + .get_raw( + history_root.as_slice().into(), + document.id().as_slice(), + None, + &new.drive.grove_version, + ) + .value + .unwrap(); + assert!(matches!(history_tree, Element::ProvableCountTree(..))); + assert_eq!(history_tree.get_flags(), legacy_tree.get_flags()); + + let mut operations = vec![]; + let delete = drive + .delete_document_for_contract_operations( + document.id(), + &contract, + document_type, + &BlockInfo::default_with_time(5_000), + Some(document_owner.into()), + None, + &mut None, + None, + new, + ) + .unwrap(); + drive + .apply_batch_low_level_drive_operations(None, None, delete, &mut operations, &new.drive) + .unwrap(); + let erase_block = BlockInfo::default_with_time(6_000); + let erase = drive + .erase_document_for_contract_operations( + document.id(), + &contract, + document_type, + &erase_block, + &mut None, + None, + new, + ) + .unwrap(); + let mut operations = vec![]; + drive + .apply_batch_low_level_drive_operations(None, None, erase, &mut operations, &new.drive) + .unwrap(); + let refunds = drive + .calculate_fee(None, Some(operations), &erase_block.epoch, new) + .unwrap() + .fee_refunds; + assert!( + drive + .grove + .get_raw( + history_root.as_slice().into(), + document.id().as_slice(), + None, + &new.drive.grove_version, + ) + .value + .is_err(), + "the erase removes the history tree" + ); + assert!(refunds.0.contains_key(&document_owner)); + assert!( + !refunds.0.contains_key(&contract_owner), + "nothing the erase removes was paid for by the contract's owner: {refunds:?}" + ); +} + +/// Two contracts holding legacy histories of every shape the migration moves: +/// a summable type, an indexed type with several revisions (two of them +/// written in one block, so one is overwritten), an immutable type, and a type +/// with no documents. +fn legacy_mixed_histories() -> (Drive, [DataContract; 2]) { + let old = PlatformVersion::get(14).unwrap(); + let drive = setup_drive_with_initial_state_structure(Some(old)); + let note = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": true, + "canBeDeleted": false, + "properties": { + "label": {"type": "string", "maxLength": 32, "position": 0}, + "rank": {"type": "integer", "minimum": 0, "maximum": 1000, "position": 1}, + }, + "required": ["label", "rank"], "additionalProperties": false, + "indices": [ + {"name": "byLabel", "properties": [{"label": "asc"}]}, + {"name": "byRank", "properties": [{"rank": "asc"}]}, + ], + }); + let tip = platform_value!({ + "type": "object", "documentsKeepHistory": true, "canBeDeleted": false, + "documentsCountable": true, "documentsSummable": "amount", + "properties": {"amount": {"type": "integer", "minimum": 0, "maximum": 4294967295i64, "position": 0}}, + "required": ["amount"], "additionalProperties": false, + "indices": [{"name": "amount", "properties": [{"amount": "asc"}]}], + }); + let stamp = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": false, + "canBeDeleted": false, + "properties": {"label": {"type": "string", "maxLength": 32, "position": 0}}, + "required": ["label"], "additionalProperties": false, + "indices": [{"name": "byLabel", "properties": [{"label": "asc"}]}], + }); + let factory = DataContractFactory::new(14).unwrap(); + let first = factory + .create_with_value_config( + [7; 32].into(), + 0, + platform_value!({"note": note.clone(), "tip": tip, "stamp": stamp, "empty": note.clone()}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + let second = factory + .create_with_value_config( + [8; 32].into(), + 1, + platform_value!({"note": note}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + for (contract, epoch) in [(&first, 1), (&second, 2)] { + drive + .apply_contract( + contract, + BlockInfo::default(), + true, + Some(Cow::Owned(StorageFlags::new_single_epoch( + epoch, + Some(contract.owner_id().to_buffer()), + ))), + None, + old, + ) + .unwrap(); + } + // (contract, type, id byte, each write's block time and values) + let histories = vec![ + ( + &first, + "note", + 1, + vec![(1_000, platform_value!({"label": "a", "rank": 1}))], + ), + ( + &first, + "note", + 2, + vec![ + (1_000, platform_value!({"label": "b", "rank": 2})), + (2_000, platform_value!({"label": "b", "rank": 3})), + (2_000, platform_value!({"label": "b2", "rank": 4})), + (3_000, platform_value!({"label": "b3", "rank": 5})), + ], + ), + ( + &first, + "note", + 3, + vec![(1_500, platform_value!({"label": "c", "rank": 2}))], + ), + ( + &first, + "tip", + 4, + vec![ + (1_000, platform_value!({"amount": 5})), + (4_000, platform_value!({"amount": 7})), + ], + ), + ( + &first, + "tip", + 5, + vec![(1_000, platform_value!({"amount": 11}))], + ), + ( + &first, + "stamp", + 6, + vec![(2_500, platform_value!({"label": "s"}))], + ), + ( + &second, + "note", + 7, + vec![ + (1_000, platform_value!({"label": "x", "rank": 9})), + (5_000, platform_value!({"label": "y", "rank": 9})), + ], + ), + ( + &second, + "note", + 8, + vec![(1_000, platform_value!({"label": "z", "rank": 0}))], + ), + ]; + for (contract, name, id, writes) in histories { + let mut document = DocumentFactory::new(14) + .unwrap() + .create_document( + contract, + contract.owner_id(), + name.into(), + writes[0].1.clone(), + ) + .unwrap(); + document.set_id([id; 32].into()); + for (index, (time, value)) in writes.into_iter().enumerate() { + if document.revision().is_some() { + document.set_revision(Some(index as u64 + 1)); + } + if let Value::Map(fields) = value { + for (field, value) in fields { + document.set(field.as_text().unwrap(), value); + } + } + add_legacy_revision( + &drive, + contract, + name, + &document, + StorageFlags::new_single_epoch((time / 1_000) as u16, Some([id + 100; 32])), + time, + ); + } + } + (drive, [first, second]) +} + +/// Every element under `path`, by its path below `path`, with each tree's +/// root key cleared: the root key records the shape of the tree under it, +/// which depends on how its writes were grouped into batches. +fn stored_elements( + drive: &Drive, + path: &[Vec], + transaction: &Transaction, +) -> BTreeMap>, Element> { + let version = PlatformVersion::latest(); + let mut elements = BTreeMap::new(); + for (key, element) in drive + .history_migration_entries(path, transaction, version) + .unwrap() + { + if !element.is_any_tree() { + elements.insert(vec![key], element); + continue; + } + let mut child = path.to_vec(); + child.push(key.clone()); + for (below, element) in stored_elements(drive, &child, transaction) { + elements.insert([vec![key.clone()], below].concat(), element); + } + elements.insert( + vec![key], + element + .reconstruct_with_root_key(None, AggregateData::NoAggregateData) + .unwrap(), + ); + } + elements +} + +/// Migrates in batches of `max_batch_operations` operations, commits, and +/// returns every element the contracts' documents tree holds afterwards. +fn migrated_elements( + drive: &Drive, + contracts: &[DataContract], + max_batch_operations: u32, +) -> ( + BTreeMap>, Element>, + DocumentHistoryMigrationStats, +) { + let mut version = PlatformVersion::latest().clone(); + version.system_limits.max_history_migration_batch_operations = Some(max_batch_operations); + let transaction = drive.grove.start_transaction(); + let stats = drive + .migrate_document_history_storage(&transaction, &version) + .unwrap(); + drive + .grove + .commit_transaction(transaction) + .unwrap() + .unwrap(); + let issues = drive + .grove + .verify_grovedb(None, true, false, &version.drive.grove_version) + .unwrap(); + assert!( + issues.is_empty(), + "batches of {max_batch_operations} operations leave inconsistent hashes: {issues:?}" + ); + let transaction = drive.grove.start_transaction(); + let mut index_entries = 0; + for contract in contracts { + for name in contract.document_types().keys() { + index_entries += assert_index_references_rewritten(drive, contract, name, &transaction); + } + } + assert_eq!(index_entries, stats.rewritten_index_entries); + let elements = stored_elements( + drive, + &[vec![RootTree::DataContractDocuments as u8]], + &transaction, + ); + (elements, stats) +} + +/// The batch size decides only how the writes are grouped, so every size +/// leaves the same elements, and GroveDB's own hashes agree with them, also +/// when a batch ends inside a type or between contracts, holds several +/// documents, or holds less than one document's writes. +#[test] +fn should_store_the_same_histories_whatever_the_batch_size() { + let default = PlatformVersion::latest() + .system_limits + .max_history_migration_batch_operations + .unwrap(); + let (drive, contracts) = legacy_mixed_histories(); + let (expected, expected_stats) = migrated_elements(&drive, &contracts, 1); + assert_eq!(expected_stats.migrated_documents, 8); + assert_eq!(expected_stats.revisions, 12); + assert_eq!(expected_stats.rewritten_index_entries, 13); + assert_eq!( + expected_stats.batches, + 5 + 8 * 2, + "one batch per history tree a type lacks, then per document its moves and its links" + ); + // Every document here writes at least seven operations, so up to seven a + // batch holds one document; from sixteen a batch holds several and can end + // inside a type; from seventy-four one pair of batches holds everything. + for (max_batch_operations, batches) in [ + (2, 17), + (7, 16), + (16, 8), + (20, 6), + (default, 2), + (u32::MAX, 2), + ] { + let (drive, contracts) = legacy_mixed_histories(); + let (elements, stats) = migrated_elements(&drive, &contracts, max_batch_operations); + assert_eq!( + elements, expected, + "batches of {max_batch_operations} operations" + ); + assert_eq!( + stats.batches, batches, + "batches of {max_batch_operations} operations" + ); + assert_eq!(stats.migrated_documents, expected_stats.migrated_documents); + assert_eq!(stats.revisions, expected_stats.revisions); + assert_eq!( + stats.rewritten_index_entries, + expected_stats.rewritten_index_entries + ); + } +} + +/// A migration that stops after some batches were applied leaves nothing +/// once its transaction is dropped, and a run in a new transaction stores +/// what an uninterrupted one does. +#[test] +fn should_store_the_same_histories_after_a_run_that_stopped_part_way() { + let (drive, contracts) = legacy_mixed_histories(); + let (reference, _) = { + let (drive, contracts) = legacy_mixed_histories(); + migrated_elements(&drive, &contracts, 1) + }; + let mut version = PlatformVersion::latest().clone(); + version.system_limits.max_history_migration_batch_operations = Some(1); + let last = contracts + .iter() + .max_by_key(|contract| contract.id()) + .unwrap(); + let last_type = contract_document_type_path_vec(last.id().as_slice(), "note"); + drive + .grove + .insert( + last_type.as_slice(), + b"unknown", + Element::empty_tree(), + None, + None, + &version.drive.grove_version, + ) + .unwrap() + .unwrap(); + let root_before = drive + .grove + .root_hash(None, &version.drive.grove_version) + .unwrap() + .unwrap(); + let transaction = drive.grove.start_transaction(); + let stopped = drive.migrate_document_history_storage(&transaction, &version); + assert!(stopped.is_err()); + assert_ne!( + drive + .grove + .root_hash(Some(&transaction), &version.drive.grove_version) + .unwrap() + .unwrap(), + root_before, + "the first contract's documents were written before the run stopped" + ); + drop(transaction); + assert_eq!( + drive + .grove + .root_hash(None, &version.drive.grove_version) + .unwrap() + .unwrap(), + root_before, + "nothing the stopped run wrote outlives its transaction" + ); + drive + .grove + .delete( + last_type.as_slice(), + b"unknown", + None, + None, + &version.drive.grove_version, + ) + .unwrap() + .unwrap(); + let (elements, stats) = migrated_elements(&drive, &contracts, 1); + assert_eq!(stats.migrated_documents, 8); + assert_eq!(elements, reference); +} diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 19ff20f0c9f..60dd8951291 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -682,6 +682,7 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, + max_history_migration_batch_operations: None, }, consensus: ConsensusVersions { tenderdash_consensus_version: 0, diff --git a/packages/rs-platform-version/src/version/system_limits/mod.rs b/packages/rs-platform-version/src/version/system_limits/mod.rs index c572525e802..5bf4c9ae048 100644 --- a/packages/rs-platform-version/src/version/system_limits/mod.rs +++ b/packages/rs-platform-version/src/version/system_limits/mod.rs @@ -568,6 +568,22 @@ pub struct SystemLimits { /// /// `None` in the tables of protocol versions that never run the rewrite. pub max_rekey_batch_moves: Option, + /// How many GroveDB operations the move of retained document histories to the composite-key + /// layout on the first block of protocol version 15 + /// (`Drive::migrate_document_history_storage`) gathers before it applies them, across + /// documents, types and contracts: one batch per document would propagate to the root once + /// per document. A document's operations are never split, so a batch holds at most one + /// document's operations more. Part of what that block writes, since the batch boundaries + /// shape the Merk trees the documents' keys are deleted from and inserted into. + /// + /// It also bounds the block's stack: a batch deletes the keys of up to about a sixth of + /// this many documents from a type's primary-key tree, and Merk's batch delete recurses + /// about once per key it removes from a tree. 500 operations stay under 512 KiB of stack + /// for a type of 32,000 documents; 10,000 need more than 2 MiB. + /// + /// `None` in the tables of protocol versions before 15, which never run the migration; every + /// table from protocol version 15 on sets it. + pub max_history_migration_batch_operations: Option, } #[cfg(test)] diff --git a/packages/rs-platform-version/src/version/system_limits/v1.rs b/packages/rs-platform-version/src/version/system_limits/v1.rs index d4c2f8c9b3b..f635693f797 100644 --- a/packages/rs-platform-version/src/version/system_limits/v1.rs +++ b/packages/rs-platform-version/src/version/system_limits/v1.rs @@ -116,4 +116,5 @@ pub const SYSTEM_LIMITS_V1: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v2.rs b/packages/rs-platform-version/src/version/system_limits/v2.rs index 9a74de90d71..0b936126554 100644 --- a/packages/rs-platform-version/src/version/system_limits/v2.rs +++ b/packages/rs-platform-version/src/version/system_limits/v2.rs @@ -97,4 +97,5 @@ pub const SYSTEM_LIMITS_V2: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v3.rs b/packages/rs-platform-version/src/version/system_limits/v3.rs index da29418d9b7..9d4b394de1b 100644 --- a/packages/rs-platform-version/src/version/system_limits/v3.rs +++ b/packages/rs-platform-version/src/version/system_limits/v3.rs @@ -99,4 +99,5 @@ pub const SYSTEM_LIMITS_V3: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v4.rs b/packages/rs-platform-version/src/version/system_limits/v4.rs index 6ef0d5f2efa..19eb8eb7200 100644 --- a/packages/rs-platform-version/src/version/system_limits/v4.rs +++ b/packages/rs-platform-version/src/version/system_limits/v4.rs @@ -220,4 +220,5 @@ pub const SYSTEM_LIMITS_V4: SystemLimits = SystemLimits { max_index_entries_per_document: Some(256), // typed array element indexes (new in v14): one document of a type with an indexed array writes at most 256 index entries max_choose_first_candidates: 1_000, // refersTo chooseFirst (new in v14): competitors a create reads, oldest first max_rekey_batch_moves: Some(10_000), // the protocol version 14 rewrite of unsigned index value keys applies its moves in batches of this many moves + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v5.rs b/packages/rs-platform-version/src/version/system_limits/v5.rs index ffe2e8d18e0..6ff2322cb40 100644 --- a/packages/rs-platform-version/src/version/system_limits/v5.rs +++ b/packages/rs-platform-version/src/version/system_limits/v5.rs @@ -22,6 +22,9 @@ use crate::version::system_limits::SystemLimits; /// * Retained-history erasure: one transition removes at most 100 revisions /// (`max_document_revisions_erased_per_transition`, `None` before), as enforced by /// `erase_document_for_contract_operations` 0. +/// * Document history layout: the first block of the version moves retained document histories +/// to the composite-key layout in batches of about 500 GroveDB operations +/// (`max_history_migration_batch_operations`, `None` before). pub const SYSTEM_LIMITS_V5: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: Some(100), // one bounded lifecycle erasure chunk (note 41) max_compiled_contract_size: Some(1_048_576), // changed: contracts stored compiled take at most 1 MiB @@ -30,5 +33,6 @@ pub const SYSTEM_LIMITS_V5: SystemLimits = SystemLimits { private_document_store_min_chunk_power: Some(4), // changed: and its chunk power is at least 4 private_document_store_max_chunk_power: Some(11), // changed: and at most 11 max_private_document_entries_per_transition: Some(16), // changed: a PrivateDocumentCreateFromShieldedPool appends at most 16 entries + max_history_migration_batch_operations: Some(500), // changed: the protocol version 15 move of retained document histories applies its writes in batches of about this many operations ..SYSTEM_LIMITS_V4 }; diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index 2a3c212d2a9..170f3539acf 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -942,7 +942,7 @@ pub const PLATFORM_V15: PlatformVersion = PlatformVersion { }, system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V4, // changed: DPNS v3 checks a name's create with keywords (note 3); the decision proposals contract (note 12); the token market contract is selected (note 17); DashPay v3 states what its data triggers checked (note 24) fee_version: FEE_VERSION4, // changed: decision proposals prefund their poll with 0.5 Dash (note 12); a private document entry pays its storage and 10,000,000 credits (note 25); a multiple choice proposal pays a 0.2 Dash processing fee on create (note 40) - system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41) + system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41); max_history_migration_batch_operations batches the move of retained document histories consensus: ConsensusVersions { tenderdash_consensus_version: 1, evidence: Some(ConsensusEvidenceParams { From 12733bcc8680ba536b92d91b7413588b5dc0272e Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 14 Nov 2026 00:16:49 +0700 Subject: [PATCH 2/3] fix(dpp): keep the document header reader in step with the full readers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A format 3 contract version stamp above u32::MAX is refused as corrupted serialization, as `from_bytes_v3` refuses it. - Every header read error keeps the underlying decoding error in its message. - Tests cover a type whose documents carry a revision only because its moderators change fields (with a moderation stamp in format 3), and formats 2 and 3 cut before, right after and inside the creator id block. - The batch-size comparison in the history migration tests keeps each tree's count and sum and clears only its root key. Test would have caught the stamp gap: should_refuse_a_contract_version_stamp_the_full_reader_refuses ✖ without the bound, ✔ with it. Co-Authored-By: Claude Opus 5.5 --- .../src/document/v0/serialize/header.rs | 145 ++++++++++++++++-- .../src/drive/document/migration/tests.rs | 13 +- 2 files changed, 137 insertions(+), 21 deletions(-) diff --git a/packages/rs-dpp/src/document/v0/serialize/header.rs b/packages/rs-dpp/src/document/v0/serialize/header.rs index 2ff9b212e81..2671f986c98 100644 --- a/packages/rs-dpp/src/document/v0/serialize/header.rs +++ b/packages/rs-dpp/src/document/v0/serialize/header.rs @@ -38,13 +38,20 @@ impl DocumentV0 { ) -> Result<(Identifier, Option), ProtocolError> { let format: FeatureVersion = serialized_document .read_varint() - .map_err(|_| header_error("serialization format"))?; + .map_err(|error| header_error("serialization format", error))?; match format { 0..=2 => {} 3 => { - let _contract_version: u64 = serialized_document + let stamp: u64 = serialized_document .read_varint() - .map_err(|_| header_error("contract version stamp"))?; + .map_err(|error| header_error("contract version stamp", error))?; + if stamp > u32::MAX as u64 { + return Err(ProtocolError::DataContractError( + DataContractError::CorruptedSerialization( + "contract version stamp does not fit in a u32".to_string(), + ), + )); + } } version => { return Err(ProtocolError::UnknownVersionMismatch { @@ -57,11 +64,11 @@ impl DocumentV0 { let mut id = [0; 32]; serialized_document .read_exact(&mut id) - .map_err(|_| header_error("id"))?; + .map_err(|error| header_error("id", error))?; let mut identifier = [0; 32]; serialized_document .read_exact(&mut identifier) - .map_err(|_| header_error("owner id"))?; + .map_err(|error| header_error("owner id", error))?; if format >= 2 && (document_type.trade_mode() != TradeMode::None || document_type.documents_transferable().is_transferable()) @@ -69,18 +76,18 @@ impl DocumentV0 { let mut has_creator_id = [0]; serialized_document .read_exact(&mut has_creator_id) - .map_err(|_| header_error("creator id flag"))?; + .map_err(|error| header_error("creator id flag", error))?; if has_creator_id[0] > 0 { serialized_document .read_exact(&mut identifier) - .map_err(|_| header_error("creator id"))?; + .map_err(|error| header_error("creator id", error))?; } } let revision = if document_type.requires_revision() { Some( serialized_document .read_varint() - .map_err(|_| header_error("revision"))?, + .map_err(|error| header_error("revision", error))?, ) } else { None @@ -89,28 +96,34 @@ impl DocumentV0 { } } -fn header_error(field: &str) -> ProtocolError { +fn header_error(field: &str, error: std::io::Error) -> ProtocolError { ProtocolError::DataContractError(DataContractError::DecodingDocumentError( DecodingError::new(format!( - "error reading the {field} from a serialized document's header" + "error reading the {field} from a serialized document's header: {error}" )), )) } #[cfg(test)] mod tests { + use crate::data_contract::config::moderation::{ContractModerationConfig, ContractModerators}; use crate::data_contract::config::DataContractConfig; + use crate::data_contract::document_type::accessors::DocumentTypeV0Getters; use crate::data_contract::document_type::methods::DocumentTypeBasicMethods; use crate::data_contract::document_type::DocumentType; + use crate::data_contract::errors::DataContractError; use crate::document::serialization_traits::DocumentPlatformConversionMethodsV0; use crate::document::v0::DocumentV0; use crate::ProtocolError; + use integer_encoding::VarInt; use platform_value::{platform_value, Identifier, Value}; use platform_version::version::PlatformVersion; use std::collections::BTreeMap; - /// A type with an object property `o` holding `members` in this order, - /// configured by `settings` (merged into the type's schema). + /// A type with an object property `o` holding `members` in this order and + /// an optional `status`, configured by `settings` (merged into the type's + /// schema), in a contract with moderators when `settings` gives them + /// abilities. fn document_type(settings: Value, members: Vec<(&str, Value)>) -> DocumentType { let platform_version = PlatformVersion::latest(); let mut schema = platform_value!({ @@ -126,19 +139,31 @@ mod tests { ), "required": ["b", "c"], "additionalProperties": false, }, + "status": {"type": "integer", "minimum": 0, "maximum": 3, "position": 1}, }, "required": ["o"], "additionalProperties": false, }); let Value::Map(settings) = settings else { panic!("type settings are a map"); }; + let mut moderated = false; for (key, value) in settings { - schema - .set_value(key.as_text().expect("a setting name"), value) - .expect("the schema is a map"); + let key = key.as_text().expect("a setting name"); + moderated |= key == "moderatorAbilities"; + schema.set_value(key, value).expect("the schema is a map"); } let config = DataContractConfig::default_for_version(platform_version).expect("a default config"); + let config = if moderated { + config.with_moderation(Some(ContractModerationConfig { + banlist: false, + suspensions: false, + moderators: ContractModerators::ContractOwner, + warnings: false, + })) + } else { + config + }; DocumentType::try_from_schema( Identifier::new([2; 32]), 1, @@ -192,6 +217,13 @@ mod tests { vec![ ("mutable", platform_value!({"documentsMutable": true})), ("immutable", platform_value!({"documentsMutable": false})), + ( + "changed by its moderators", + platform_value!({ + "documentsMutable": false, + "moderatorAbilities": {"changeFields": ["status"]}, + }), + ), ( "transferable", platform_value!({"documentsMutable": false, "transferable": 1}), @@ -220,10 +252,29 @@ mod tests { let platform_version = PlatformVersion::latest(); for (name, settings) in configurations() { let document_type = document_type(settings, members(&["b", "c"])); + let moderated = document_type.as_ref().has_moderator_changeable_fields(); + if moderated { + assert!( + !document_type.as_ref().documents_mutable() + && !document_type + .as_ref() + .documents_transferable() + .is_transferable() + && document_type.as_ref().requires_revision(), + "the moderators' fields alone give the {name} type a revision" + ); + } let revision = document_type.as_ref().requires_revision().then_some(4); for creator_id in [None, Some([3; 32])] { for format in 0..=3 { - let bytes = serialize(&document(revision, creator_id), format, &document_type); + let mut document = document(revision, creator_id); + if moderated && format == 3 { + // The moderation stamp sits in the time fields, after the + // revision; only format 3 has a place for it. + document.moderated_at = Some(2_000); + document.moderated_by = Some(Identifier::new([5; 32])); + } + let bytes = serialize(&document, format, &document_type); let decoded = DocumentV0::from_bytes(&bytes, document_type.as_ref(), platform_version) .expect("bytes the type reads"); @@ -286,4 +337,66 @@ mod tests { Err(ProtocolError::UnknownVersionMismatch { received: 4, .. }) )); } + + #[test] + fn should_refuse_a_header_cut_inside_its_creator_id_block() { + let document_type = document_type( + platform_value!({"documentsMutable": false, "transferable": 1}), + members(&["b", "c"]), + ); + for format in [2, 3] { + let bytes = serialize(&document(Some(4), Some([3; 32])), format, &document_type); + // The format, an unset contract version stamp in format 3, the id + // and the owner id come before the creator id flag. + let flag = if format == 3 { 66 } else { 65 }; + assert_eq!(bytes[flag], 1, "format {format} marks a creator id"); + for (end, cut) in [ + (flag, "before the creator id flag"), + (flag + 1, "right after the creator id flag"), + (flag + 17, "inside the creator id"), + (flag + 33, "before the revision"), + ] { + assert!( + DocumentV0::id_and_revision_from_bytes(&bytes[..end], document_type.as_ref()) + .is_err(), + "format {format} cut {cut} holds no complete header" + ); + } + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes[..flag + 34], document_type.as_ref()) + .expect("a complete header"), + (Identifier::new([9; 32]), Some(4)), + "format {format}" + ); + } + } + + #[test] + fn should_refuse_a_contract_version_stamp_the_full_reader_refuses() { + let platform_version = PlatformVersion::latest(); + let document_type = document_type( + platform_value!({"documentsMutable": true}), + members(&["b", "c"]), + ); + let bytes = serialize(&document(Some(4), None), 3, &document_type); + assert_eq!(bytes[1], 0, "an unset stamp is one byte"); + let with_stamp = |stamp: u64| [&bytes[..1], &stamp.encode_var_vec(), &bytes[2..]].concat(); + let largest = with_stamp(u32::MAX as u64); + assert!(DocumentV0::from_bytes(&largest, document_type.as_ref(), platform_version).is_ok()); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&largest, document_type.as_ref()) + .expect("a stamp that fits in a u32"), + (Identifier::new([9; 32]), Some(4)) + ); + let too_large = with_stamp(u32::MAX as u64 + 1); + assert!( + DocumentV0::from_bytes(&too_large, document_type.as_ref(), platform_version).is_err() + ); + assert!(matches!( + DocumentV0::id_and_revision_from_bytes(&too_large, document_type.as_ref()), + Err(ProtocolError::DataContractError( + DataContractError::CorruptedSerialization(_) + )) + )); + } } diff --git a/packages/rs-drive/src/drive/document/migration/tests.rs b/packages/rs-drive/src/drive/document/migration/tests.rs index 471c35af85c..cf66bf8f795 100644 --- a/packages/rs-drive/src/drive/document/migration/tests.rs +++ b/packages/rs-drive/src/drive/document/migration/tests.rs @@ -1224,8 +1224,9 @@ fn legacy_mixed_histories() -> (Drive, [DataContract; 2]) { } /// Every element under `path`, by its path below `path`, with each tree's -/// root key cleared: the root key records the shape of the tree under it, -/// which depends on how its writes were grouped into batches. +/// root key cleared and its count and sum kept: the root key records the +/// shape of the tree under it, which depends on how its writes were grouped +/// into batches. fn stored_elements( drive: &Drive, path: &[Vec], @@ -1246,11 +1247,13 @@ fn stored_elements( for (below, element) in stored_elements(drive, &child, transaction) { elements.insert([vec![key.clone()], below].concat(), element); } + let aggregate = AggregateData::CountAndSum( + element.count_value_or_default(), + element.sum_value_or_default(), + ); elements.insert( vec![key], - element - .reconstruct_with_root_key(None, AggregateData::NoAggregateData) - .unwrap(), + element.reconstruct_with_root_key(None, aggregate).unwrap(), ); } elements From b9f2d6a81b95271faee6fa6352f93e929a319ba6 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 14 Nov 2026 00:37:50 +0700 Subject: [PATCH 3/3] fix(platform)!: reserve an erase's refund work in check tx and size its revisions without overflow (PV15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to the keep-history document erase, which only protocol version 15 runs. - drive-abci: check tx validates no state of an unsponsored batch, while the block's state validation bills every erase for the work its refunds cause for other identities (`erase_refund_recipient_cost`). Check tx now reserves that work for each erase of such a batch, on the first check and on every recheck, next to the `chooseFirst` scan budget it already reserves for creates (one `reserve_unvalidated_state_work` for both). The reservation follows erase state validation 0 and refuses any other generation, which would have to reserve what it bills. Block execution is unchanged. - dpp: `DocumentType::max_size` 1 sizes each property with `saturating_max_byte_size`, so a string of 16384 or more characters without `maxBytes` counts as `u16::MAX` bytes instead of failing with an overflow. `CONTRACT_VERSIONS_V7` (protocol version 15) selects it; only the erase of retained revisions reads `max_size`, and protocol version 14 keeps 0. The erase now sizes its revisions only when it estimates. - The erase accounting in the book and the protocol version 15 changelog describe both. Tests would have caught both in CI: - should_reserve_the_refund_recipient_work_when_admitting_an_erase ✖ before (the check tx estimate, 35,771,120 credits of processing, held none of the 446,933,400 the erase is billed for; at a balance of that estimate the block refused the erase unpaid), ✔ after: the first check's estimate and the lifecycle read only state validation makes cover the block's estimate. - should_erase_a_type_whose_string_bound_does_not_fit_a_document_size ✖ before (check tx failed with "max_byte_size overflow"), ✔ after, with the admitting estimate covering the fee charged for a full chunk of the largest revisions. Co-Authored-By: Claude Opus 5.5 --- book/src/drive/keep-history-documents.md | 7 +- .../document_type/methods/mod.rs | 84 +++++++- .../methods/versioned_methods.rs | 21 ++ .../check_tx_verification/v0/mod.rs | 42 ++-- .../document_erase_transition_action/mod.rs | 2 + .../state_v0/mod.rs | 68 +++++++ .../state_transitions/batch/mod.rs | 3 + .../batch/tests/document/erase.rs | 187 +++++++++++++++++- .../note-contract-keep-history-erasable.json | 19 ++ .../v0/mod.rs | 9 +- .../dpp_versions/dpp_contract_versions/v7.rs | 7 +- .../rs-platform-version/src/version/v15.rs | 7 +- 12 files changed, 429 insertions(+), 27 deletions(-) diff --git a/book/src/drive/keep-history-documents.md b/book/src/drive/keep-history-documents.md index 4507f0f38bb..3bb91d97fc7 100644 --- a/book/src/drive/keep-history-documents.md +++ b/book/src/drive/keep-history-documents.md @@ -196,7 +196,12 @@ largest documents plus both endings; every erase is admitted against the same worst-case estimate and the actual fee is what it removed. GroveDB's query surface has no key-only result shape over a range, so the enumeration reads revision bodies it discards, and the estimate charges for that read -honestly. +honestly. The type's largest document is sized with each string bound past +`u16::MAX` bytes held at `u16::MAX`. The balance updates the refunds cause +for other identities are billed on top, to the identity submitting the +erase, for a full chunk of recipients whatever the erase removes; `check_tx` +holds the submitter to that charge as well, though it does not validate the +erase's state. ## The `getDocumentHistory` query diff --git a/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs b/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs index 0140e1fa03a..196d1ea55a2 100644 --- a/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs @@ -1129,9 +1129,10 @@ pub trait DocumentTypeV0Methods: DocumentTypeV0Getters + DocumentTypeV0MethodsVe .max_size { 0 => self.max_size_v0(platform_version), + 1 => self.max_size_v1(platform_version), version => Err(ProtocolError::UnknownVersionMismatch { method: "max_size".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, }), } @@ -1699,6 +1700,87 @@ mod tests { ); } + // -------------------------------------------------------------- + // DocumentTypeV0Methods::max_size + // -------------------------------------------------------------- + + /// A string of 20000 characters without `maxBytes` counts as `u16::MAX` + /// bytes toward the largest size, as a string without `maxLength` does, + /// from protocol version 15. Generation 0, which protocol version 14 + /// selects, fails on the overflow. + #[test] + fn should_hold_a_string_past_16383_characters_at_u16_max_in_the_largest_size() { + let platform_version = PlatformVersion::latest(); + let long = build_doc_type("note", note_schema(long_text())); + let unbounded = build_doc_type( + "note", + note_schema(platform_value!({"type": "string", "position": 0})), + ); + assert_eq!( + long.as_ref() + .max_size(platform_version) + .expect("the long string is sized"), + u16::MAX + ); + assert_eq!( + unbounded + .as_ref() + .max_size(platform_version) + .expect("the unbounded string is sized"), + u16::MAX + ); + + let platform_version_14 = PlatformVersion::get(14).expect("expected version 14"); + let long_14 = build_doc_type_at("note", note_schema(long_text()), platform_version_14); + assert!(matches!( + long_14.as_ref().max_size(platform_version_14), + Err(ProtocolError::Overflow(_)) + )); + } + + /// Below the cap both generations size every property alike, an object's + /// members and a typed array's elements included. + #[test] + fn should_size_bounded_properties_as_protocol_version_14_does() { + let platform_version_14 = PlatformVersion::get(14).expect("expected version 14"); + let schema = platform_value!({ + "type": "object", + "properties": { + "title": {"type": "string", "minLength": 3, "maxLength": 100, "position": 0}, + "count": {"type": "integer", "minimum": 0, "maximum": 1000, "position": 1}, + "data": {"type": "array", "byteArray": true, "maxItems": 64, "position": 2}, + "meta": { + "type": "object", + "properties": { + "name": {"type": "string", "maxLength": 10, "position": 0}, + "rank": {"type": "integer", "minimum": 0, "maximum": 9, "position": 1}, + }, + "additionalProperties": false, + "position": 3, + }, + "tags": { + "type": "array", + "items": {"type": "string", "maxLength": 10}, + "maxItems": 4, + "position": 4, + }, + }, + "additionalProperties": false, + }); + let at_latest = build_doc_type("doc", schema.clone()); + let at_14 = build_doc_type_at("doc", schema, platform_version_14); + let size = at_latest + .as_ref() + .max_size(PlatformVersion::latest()) + .expect("sized"); + assert_eq!( + size, + at_14.as_ref().max_size(platform_version_14).expect("sized") + ); + // The title alone is 400 bytes, and nothing comes near the cap + assert!(size > 400 && size < 1024, "{size}"); + } + // -------------------------------------------------------------- // DocumentTypeBasicMethods::requires_revision / initial_revision // -------------------------------------------------------------- diff --git a/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs b/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs index eba6769dfa4..f64e8af282a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs +++ b/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs @@ -700,6 +700,27 @@ pub trait DocumentTypeV0MethodsVersioned: DocumentTypeV0Getters + DocumentTypeBa Ok(total_size) } + /// Generation 0 with each property sized by + /// [`DocumentPropertyType::saturating_max_byte_size`]: a string of 16384 + /// or more characters without `maxBytes` has a byte bound past `u16::MAX`, + /// which generation 0 fails on with an overflow and this generation holds + /// at `u16::MAX`. Every other type is sized as generation 0 sizes it, and + /// the total saturates as generation 0's does. + fn max_size_v1(&self, platform_version: &PlatformVersion) -> Result { + let mut total_size = 0u16; + + for document_property in self.flattened_properties().values() { + if let Some(size) = document_property + .property_type + .saturating_max_byte_size(platform_version)? + { + total_size = total_size.saturating_add(size); + } + } + + Ok(total_size) + } + /// Figures out the prefunded voting balance (v0) for a document in a document type fn prefunded_voting_balance_for_document_v0( &self, diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs index 48171b1ac7c..81cfcc985ef 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs @@ -21,6 +21,7 @@ use crate::execution::types::state_transition_execution_context::{ StateTransitionExecutionContext, StateTransitionExecutionContextMethodsV0, }; use crate::execution::validation::state_transition::batch::choose_first_reservation; +use crate::execution::validation::state_transition::batch::erase_refund_recipient_reservation; use crate::execution::validation::state_transition::common::asset_lock::proof::verify_is_not_spent::AssetLockProofVerifyIsNotSpent; use crate::execution::validation::state_transition::processor::address_witnesses::{StateTransitionAddressWitnessValidationV0, StateTransitionHasAddressWitnessValidationV0}; use crate::execution::validation::state_transition::processor::traits::shielded_proof::{ @@ -533,14 +534,15 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } // A create choosing the first of the commitments to its values reads up to its scan's - // budget of competitors in a block; its state was not validated here, so the signer - // is held to that budget too + // budget of competitors in a block, and an erase is billed for the work its refunds + // cause; their state was not validated here, so the signer is held to both too if !validates_state { - reserve_choose_first_scans( + reserve_unvalidated_state_work( &action, + platform, &mut state_transition_execution_context, platform_version, - ); + )?; } let execution_event = ExecutionEvent::create_from_state_transition_action( @@ -741,13 +743,15 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } } - // As on the first check: the budget of a scan whose state was not validated + // As on the first check: the budget of a scan and the refund work of an erase + // whose state was not validated if !relies_on_gas_sponsor_to_pay { - reserve_choose_first_scans( + reserve_unvalidated_state_work( &action, + platform, &mut state_transition_execution_context, platform_version, - ); + )?; } let execution_event = ExecutionEvent::create_from_state_transition_action( @@ -771,18 +775,26 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } } -/// Reserves, on top of the estimated fee of `action`, what the `chooseFirst` checks of its -/// creates may bill in a block (see `choose_first_reservation`: their two hashes and the -/// competitor scan's budget), for an action whose state check tx did not validate. Fee +/// Reserves, on top of the estimated fee of `action`, what state validation bills in a block +/// for an action whose state check tx did not validate: what the `chooseFirst` checks of its +/// creates may bill (see `choose_first_reservation`: their two hashes and the competitor scan's +/// budget) and the work the refunds of its erases cause for other identities (see +/// `erase_refund_recipient_reservation`), priced at the epoch check tx estimates in. Fee /// validation then admits only a signer who could pay for them. -fn reserve_choose_first_scans( +fn reserve_unvalidated_state_work( action: &StateTransitionAction, + platform: &PlatformRef, execution_context: &mut StateTransitionExecutionContext, platform_version: &PlatformVersion, -) { - for operation in choose_first_reservation(action, platform_version) { - execution_context.add_operation(operation); - } +) -> Result<(), Error> { + execution_context.add_operations(choose_first_reservation(action, platform_version)); + execution_context.add_operations(erase_refund_recipient_reservation( + action, + platform.drive, + platform.state.last_committed_block_epoch_ref(), + platform_version, + )?); + Ok(()) } #[cfg(test)] diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs index a98788c0d28..774d8162e78 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs @@ -14,6 +14,8 @@ use drive::state_transition_action::batch::batched_transition::document_transiti mod advanced_structure_v0; mod state_v0; +pub(in crate::execution::validation::state_transition) use state_v0::erase_refund_recipient_reservation; + pub trait DocumentEraseTransitionActionValidation { fn validate_structure( &self, diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs index 6285f88b77c..047214547fb 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs @@ -1,3 +1,4 @@ +use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::execution_operation::ValidationOperation; use crate::execution::types::state_transition_execution_context::{ @@ -7,6 +8,7 @@ use crate::execution::validation::state_transition::batch::action_validation::do use crate::execution::validation::state_transition::batch::state::fetch_keep_history_document_lifecycle::fetch_keep_history_document_lifecycle; use crate::platform_types::platform::PlatformStateRef; use dpp::block::block_info::BlockInfo; +use dpp::block::epoch::Epoch; use dpp::consensus::basic::document::{ InvalidDocumentTransitionActionError, InvalidDocumentTypeError, }; @@ -20,7 +22,11 @@ use dpp::identifier::Identifier; use dpp::validation::SimpleConsensusValidationResult; use dpp::version::PlatformVersion; use drive::drive::document::lifecycle::DocumentLifecycleState; +use drive::drive::Drive; use drive::grovedb::TransactionArg; +use drive::state_transition_action::batch::batched_transition::document_transition::DocumentTransitionAction; +use drive::state_transition_action::batch::batched_transition::BatchedTransitionAction; +use drive::state_transition_action::StateTransitionAction; use drive::state_transition_action::batch::batched_transition::document_transition::document_base_transition_action::DocumentBaseTransitionActionAccessorsV0; use drive::state_transition_action::batch::batched_transition::document_transition::document_erase_transition_action::v0::DocumentEraseTransitionActionAccessorsV0; use drive::state_transition_action::batch::batched_transition::document_transition::document_erase_transition_action::DocumentEraseTransitionAction; @@ -141,3 +147,65 @@ impl DocumentEraseTransitionActionStateValidationV0 for DocumentEraseTransitionA } } } + +/// What check tx reserves, on top of the estimated fee of `action`, for the +/// erases of a batch whose state it does not validate: for every erase, the +/// work its refunds cause for other identities +/// ([`Drive::erase_refund_recipient_cost`]), which state validation 0 bills on +/// every path an erase takes. Nothing for anything else. Without it, a signer +/// covering an erase's estimated fee but not that work would be admitted, then +/// refused unpaid in a block, its nonce left to replay the work. A state +/// validation generation that bills an erase otherwise has to reserve what it +/// bills, so any other generation is refused here. +pub(in crate::execution::validation::state_transition) fn erase_refund_recipient_reservation( + action: &StateTransitionAction, + drive: &Drive, + epoch: &Epoch, + platform_version: &PlatformVersion, +) -> Result, Error> { + let StateTransitionAction::BatchAction(batch) = action else { + return Ok(Vec::new()); + }; + let erases = batch + .transitions() + .iter() + .filter(|transition| { + matches!( + transition, + BatchedTransitionAction::DocumentAction(DocumentTransitionAction::EraseAction(_)) + ) + }) + .count(); + if erases == 0 { + return Ok(Vec::new()); + } + match platform_version + .drive_abci + .validation_and_processing + .state_transitions + .batch_state_transition + .document_erase_transition_state_validation + { + Some(0) => {} + Some(version) => { + return Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "erase_refund_recipient_reservation".to_string(), + known_versions: vec![0], + received: version, + })) + } + None => { + return Err(Error::Execution(ExecutionError::VersionNotActive { + method: "erase_refund_recipient_reservation".to_string(), + known_versions: vec![0], + })) + } + } + let cost = drive + .erase_refund_recipient_cost(epoch, platform_version) + .map_err(Error::Drive)?; + Ok(vec![ + ValidationOperation::PrecalculatedOperation(cost); + erases + ]) +} diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs index 20d2fad4500..9a91f8daa04 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs @@ -15,6 +15,9 @@ pub(in crate::execution::validation::state_transition) use state::v0::fetch_docu // Check tx reserves, for a create it does not validate the state of, what the block's // `chooseFirst` check may bill it: its two hashes and the competitor scan's budget. pub(in crate::execution::validation::state_transition) use action_validation::document::document_reference_validation::v0::choose_first_reservation; +// Check tx reserves, for an erase it does not validate the state of, the work its refunds cause +// for other identities, which the block's state validation bills. +pub(in crate::execution::validation::state_transition) use action_validation::document::document_erase_transition_action::erase_refund_recipient_reservation; // Moderator field changes and restores judge the document's `propertyConstraints`, with // the totals after the replace or insertion respectively. pub(in crate::execution::validation::state_transition) use transformer::v0::property_constraint_aggregates::{ diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs index 92f385774ab..5de5908a9be 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs @@ -5,14 +5,21 @@ //! so an owner who loses their keys, their funds or their permission cannot //! strand a half-erased document. +use super::gas_sponsorship::gas_sponsorship_tests::IDENTITY_INSUFFICIENT_BALANCE; use super::*; +use crate::execution::check_tx::CheckTxLevel; +use crate::execution::check_tx::CheckTxLevel::{FirstTimeCheck, Recheck}; +use crate::platform_types::platform::PlatformRef; use crate::rpc::core::MockCoreRPCLike; use crate::test::helpers::setup::TempPlatform; +use dpp::block::epoch::Epoch; use dpp::consensus::basic::BasicError; +use dpp::consensus::codes::ErrorWithCode; use dpp::data_contract::document_type::accessors::DocumentTypeV2Getters; use dpp::data_contract::document_type::DocumentTypeRef; use dpp::data_contract::DataContract; use dpp::document::Document; +use dpp::fee::fee_result::FeeResult; use dpp::identifier::Identifier; use dpp::identity::{Identity, IdentityPublicKey, SecurityLevel}; use dpp::prelude::IdentityNonce; @@ -90,6 +97,12 @@ struct Fixture { impl Fixture { async fn new(document_type_name: &str) -> Self { + Self::new_with_message(document_type_name, None).await + } + + /// As [`Fixture::new`], with the document's `message` set to `message` + /// rather than a random one when given. + async fn new_with_message(document_type_name: &str, message: Option<&str>) -> Self { let platform_version = PlatformVersion::latest(); let mut platform = TestPlatformBuilder::new() .with_initial_protocol_version(platform_version.protocol_version) @@ -130,6 +143,9 @@ impl Fixture { platform_version, ) .expect("expected a random note"); + if let Some(message) = message { + document.set("message", message.into()); + } document .set_id_for_creation(document_type, &entropy.0, 1, platform_version) @@ -403,6 +419,61 @@ impl Fixture { .state } + /// The codes of the errors check tx at `level` refuses `transition` with, + /// and the fee it estimated when it got that far. + fn check_tx_at(&self, transition: &[u8], level: CheckTxLevel) -> (Vec, Option) { + let state = self.platform.state.load(); + let platform_ref = PlatformRef { + drive: &self.platform.drive, + state: &state, + config: &self.platform.config, + core_rpc: &self.platform.core_rpc, + }; + let result = self + .platform + .check_tx(transition, level, &platform_ref, PlatformVersion::latest()) + .expect("expected to check the transaction"); + let codes = result.errors.iter().map(|error| error.code()).collect(); + (codes, result.data.and_then(|check| check.fee_result)) + } + + /// Sets the owner's balance to `credits`. + fn set_owner_balance(&self, credits: Credits) { + let platform_version = PlatformVersion::latest(); + let id = self.owner.id().to_buffer(); + let balance = self + .platform + .drive + .fetch_identity_balance(id, None, platform_version) + .expect("expected to fetch the balance") + .expect("expected a balance"); + if balance > credits { + self.platform + .drive + .remove_from_identity_balance( + id, + balance - credits, + &BlockInfo::default(), + true, + None, + platform_version, + ) + .expect("expected to lower the balance"); + } else if credits > balance { + self.platform + .drive + .add_to_identity_balance( + id, + credits - balance, + &BlockInfo::default(), + true, + None, + platform_version, + ) + .expect("expected to raise the balance"); + } + } + fn lifecycle(&self, document_type_name: &str) -> DocumentLifecycleState { self.platform .drive @@ -882,8 +953,6 @@ async fn should_delete_and_erase_in_one_block() { /// in the estimate that admits the transition and in the fee actually charged. #[tokio::test] async fn should_charge_an_erase_for_the_refund_recipients_it_can_credit() { - use dpp::block::epoch::Epoch; - let mut fixture = Fixture::new("note").await; assert_successful(&fixture.delete_as_owner("note").await, "the delete"); @@ -1245,3 +1314,117 @@ async fn should_allow_the_new_owner_to_delete_and_erase_a_transferred_history() DocumentLifecycleState::Absent ); } + +/// Check tx validates no state of an unsponsored erase, and the work its +/// refunds cause for other identities is billed there, so check tx reserves +/// that work on top of the estimate, on the first check and on every recheck. +/// A signer one credit short of the estimate so reserved, who covers the +/// estimate without the reservation, stays out of the mempool rather than +/// being refused unpaid in a block; one holding the estimate is admitted. The +/// estimate the block admits the erase against is then the first check's and +/// the lifecycle read only state validation makes. +#[tokio::test] +async fn should_reserve_the_refund_recipient_work_when_admitting_an_erase() { + let mut fixture = Fixture::new("note").await; + assert_successful(&fixture.delete_as_owner("note").await, "the delete"); + let erase = fixture.erase_transition("note", true).await; + let recipient_cost = fixture + .platform + .drive + .erase_refund_recipient_cost(&Epoch::new(0).unwrap(), PlatformVersion::latest()) + .expect("expected the recipient work to be priced"); + + let mut first_check_estimate = None; + for level in [FirstTimeCheck, Recheck] { + fixture.set_owner_balance(dash_to_credits!(1.0)); + let (errors, estimated) = fixture.check_tx_at(&erase, level); + assert_eq!(errors, Vec::::new(), "{level:?}"); + let estimated = estimated.expect("expected an estimated fee"); + assert!( + estimated.processing_fee >= recipient_cost.processing_fee, + "{level:?}: the estimate holds the recipient work: {} is below {}", + estimated.processing_fee, + recipient_cost.processing_fee + ); + let required = estimated.total_base_fee(); + first_check_estimate.get_or_insert(required); + + fixture.set_owner_balance(required - 1); + assert_eq!( + fixture.check_tx_at(&erase, level).0, + vec![IDENTITY_INSUFFICIENT_BALANCE], + "{level:?}" + ); + fixture.set_owner_balance(required); + assert_eq!( + fixture.check_tx_at(&erase, level).0, + Vec::::new(), + "{level:?}" + ); + } + + let (_, lifecycle_read) = fixture + .platform + .drive + .fetch_document_lifecycle( + &fixture.contract, + fixture.document_type("note"), + fixture.document.id(), + Some(&Epoch::new(0).unwrap()), + None, + PlatformVersion::latest(), + ) + .expect("expected to read the lifecycle"); + fixture.set_owner_balance(dash_to_credits!(1.0)); + let result = process(&mut fixture.platform, erase); + let StateTransitionExecutionResult::SuccessfulExecution { estimated_fees, .. } = result else { + panic!("expected the erase to succeed, got {result:?}"); + }; + let block_estimate = estimated_fees + .expect("an erase is admitted against an estimate") + .total_base_fee(); + let first_check_estimate = first_check_estimate.expect("the first check estimated a fee"); + assert!( + first_check_estimate + lifecycle_read.total_base_fee() >= block_estimate, + "check tx admits the erase against all the block does but read its lifecycle: \ + {first_check_estimate} and a read of {} is below {block_estimate}", + lifecycle_read.total_base_fee() + ); +} + +/// A type whose string may hold 16384 characters and declares no `maxBytes` +/// has a byte bound past `u16::MAX` at four bytes a character. An erase sizes +/// the revisions it may remove as the type's largest documents, held at +/// `u16::MAX`, so it runs and is charged, and the estimate that admits it +/// covers a full chunk of revisions as large as a document's fields may be. +#[tokio::test] +async fn should_erase_a_type_whose_string_bound_does_not_fit_a_document_size() { + let platform_version = PlatformVersion::latest(); + let longest_message = "x".repeat(platform_version.system_limits.max_field_value_size as usize); + let mut fixture = Fixture::new_with_message("longNote", Some(&longest_message)).await; + let chunk = platform_version + .system_limits + .max_document_revisions_erased_per_transition + .expect("protocol 15 bounds the chunk") as u64; + fixture.retain_revisions("longNote", chunk); + assert_successful(&fixture.delete_as_owner("longNote").await, "the delete"); + let erase = fixture.erase_transition("longNote", true).await; + + let (errors, estimated) = fixture.check_tx_at(&erase, FirstTimeCheck); + assert_eq!(errors, Vec::::new()); + let estimated = estimated.expect("expected an estimated fee"); + let result = process(&mut fixture.platform, erase); + let StateTransitionExecutionResult::SuccessfulExecution { fee_result, .. } = result else { + panic!("expected the erase to succeed, got {result:?}"); + }; + assert!( + estimated.total_base_fee() >= fee_result.total_base_fee(), + "the estimate that admits the erase covers what it is charged: {} is below {}", + estimated.total_base_fee(), + fee_result.total_base_fee() + ); + assert_matches!( + fixture.lifecycle("longNote"), + DocumentLifecycleState::Absent + ); +} diff --git a/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json b/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json index 6c5816fe4ff..898ba7438e1 100644 --- a/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json +++ b/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json @@ -61,6 +61,25 @@ "additionalProperties": false, "$comment": "A keep-history type whose deleted documents may also have their retained revisions purged. Deleting a note removes it from ordinary reads and leaves its revisions readable; erasing removes those revisions a bounded chunk at a time.", "transferable": 1 + }, + "longNote": { + "type": "object", + "documentsKeepHistory": true, + "documentsMutable": true, + "canBeDeleted": true, + "canBeErased": true, + "properties": { + "message": { + "type": "string", + "maxLength": 16384, + "position": 0 + } + }, + "required": [ + "message" + ], + "additionalProperties": false, + "$comment": "An erasable note whose message may hold 16384 characters and declares no maxBytes: at four bytes a character its byte bound does not fit in the u16 a document type's sizes are computed in." } }, "groups": {}, diff --git a/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs index 8060dd2afe6..6c38499511a 100644 --- a/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs @@ -94,14 +94,13 @@ impl Drive { contract.id_ref().as_bytes(), document_type.name().as_str(), ); - // Worst-case sizing: a chunk may hold the type's largest documents, and - // the admission estimate has to cover that whatever this document - // actually stores. - let revision_size = document_type.max_size(platform_version)? as u32; - let mut batch_operations: Vec = vec![]; if let Some(layers) = estimated_costs_only_with_layer_info { + // Worst-case sizing: a chunk may hold the type's largest documents, + // and the admission estimate has to cover that whatever this + // document actually stores. + let revision_size = document_type.max_size(platform_version)? as u32; Self::add_estimation_costs_for_erase_document( document_id, contract, diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs index c58b4b21b62..170d1b18ee3 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs @@ -46,7 +46,10 @@ use crate::version::dpp_versions::dpp_contract_versions::{ /// format version 1; /// - `canonical_contested_index_values`, `Some(1)` rather than `Some(0)`: every index value /// of a contest is written the one way its property's type keys it, so the values of one -/// contest name one poll; 0 wrote only identifiers one way. +/// contest name one poll; 0 wrote only identifiers one way; +/// - `max_size`, 1 rather than 0: a string of 16384 or more characters without `maxBytes` +/// counts as `u16::MAX` bytes toward a document type's largest size rather than failing it +/// with an overflow. Only the erase of retained revisions, new in this version, reads it. /// /// v6 remains unchanged for `PROTOCOL_VERSION_14`, whose parse refuses a byte array key and /// whose schemas and documents the `jsonschema` crate checks. @@ -91,6 +94,8 @@ pub const CONTRACT_VERSIONS_V7: DPPContractVersions = DPPContractVersions { // changed: every index value of a contest is written the one way its property's // type keys it, where 0 wrote only identifiers one way canonical_contested_index_values: Some(1), + // changed: a string bound past `u16::MAX` bytes is held there, not an overflow + max_size: 1, ..CONTRACT_VERSIONS_V6.document_type_versions.methods }, ..CONTRACT_VERSIONS_V6.document_type_versions diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index 170f3539acf..5bca17022e7 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -892,7 +892,10 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// deletion time, and an erasing or erased document is absent at every requested time. /// Parser 4 admits `canBeErased` with meta-schema 3; compiled type storage keeps generation /// 0 and its existing non-lifecycle bytes. Protocols through 14 retain their handlers, -/// limits and wire bounds, and cannot admit an erase. +/// limits and wire bounds, and cannot admit an erase. An erase sizes the revisions it may +/// remove with `max_size` 1 (`CONTRACT_VERSIONS_V7`), which counts a string of 16384 or +/// more characters without `maxBytes` as `u16::MAX` bytes where generation 0 fails on the +/// overflow. /// 42. **A contract's patterns are bounded**: `CONTRACT_VERSIONS_V7` selects /// `validate_schema_patterns` 0. Contract registration and update refuse a contract whose /// document schemas and `$defs` hold more than 32 patterns @@ -931,7 +934,7 @@ pub const PLATFORM_V15: PlatformVersion = PlatformVersion { state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V2, state_transitions: STATE_TRANSITION_VERSIONS_V5, // changed: validate_asset_lock_transaction_structure 1 drops the input cap and validate_instant_asset_lock_proof_structure 1 applies it to instant proofs only, so a chain proof redeems a lock of more than 100 inputs (note 30) - contract_versions: CONTRACT_VERSIONS_V7, // changed: parse_property_constraints 1 lets countOf and sumOf match byte array keys (note 4); validate_document 2 checks documents natively (note 6); check_document_type_schema 1 checks schemas natively at registration (note 7); contract_storage_serialization_version 2 stores contracts compiled (note 8); parse_private_document 0 parses private document types (note 13); apply_generated_from 1 admits sys.tokens.decimals (note 15); try_from_schema 4 parses lifecycle flags (note 41); validate_schema_patterns 0 bounds the patterns of a contract and registration_cost 3 estimates its fee without compiling one (note 42) + contract_versions: CONTRACT_VERSIONS_V7, // changed: parse_property_constraints 1 lets countOf and sumOf match byte array keys (note 4); validate_document 2 checks documents natively (note 6); check_document_type_schema 1 checks schemas natively at registration (note 7); contract_storage_serialization_version 2 stores contracts compiled (note 8); parse_private_document 0 parses private document types (note 13); apply_generated_from 1 admits sys.tokens.decimals (note 15); try_from_schema 4 parses lifecycle flags (note 41); max_size 1 holds a string bound past u16::MAX bytes there for the erase (note 41); validate_schema_patterns 0 bounds the patterns of a contract and registration_cost 3 estimates its fee without compiling one (note 42) document_versions: DOCUMENT_VERSIONS_V4, identity_versions: IDENTITY_VERSIONS_V1, voting_versions: VOTING_VERSION_V3, // changed: the yes/no poll's stored info, id, parameter rules and decision rules are active (note 10), and the decision proposal's poll (note 12)