diff --git a/book/src/drive/keep-history-documents.md b/book/src/drive/keep-history-documents.md index 4507f0f38bb..3bb91d97fc7 100644 --- a/book/src/drive/keep-history-documents.md +++ b/book/src/drive/keep-history-documents.md @@ -196,7 +196,12 @@ largest documents plus both endings; every erase is admitted against the same worst-case estimate and the actual fee is what it removed. GroveDB's query surface has no key-only result shape over a range, so the enumeration reads revision bodies it discards, and the estimate charges for that read -honestly. +honestly. The type's largest document is sized with each string bound past +`u16::MAX` bytes held at `u16::MAX`. The balance updates the refunds cause +for other identities are billed on top, to the identity submitting the +erase, for a full chunk of recipients whatever the erase removes; `check_tx` +holds the submitter to that charge as well, though it does not validate the +erase's state. ## The `getDocumentHistory` query diff --git a/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs b/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs index 0140e1fa03a..196d1ea55a2 100644 --- a/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/methods/mod.rs @@ -1129,9 +1129,10 @@ pub trait DocumentTypeV0Methods: DocumentTypeV0Getters + DocumentTypeV0MethodsVe .max_size { 0 => self.max_size_v0(platform_version), + 1 => self.max_size_v1(platform_version), version => Err(ProtocolError::UnknownVersionMismatch { method: "max_size".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, }), } @@ -1699,6 +1700,87 @@ mod tests { ); } + // -------------------------------------------------------------- + // DocumentTypeV0Methods::max_size + // -------------------------------------------------------------- + + /// A string of 20000 characters without `maxBytes` counts as `u16::MAX` + /// bytes toward the largest size, as a string without `maxLength` does, + /// from protocol version 15. Generation 0, which protocol version 14 + /// selects, fails on the overflow. + #[test] + fn should_hold_a_string_past_16383_characters_at_u16_max_in_the_largest_size() { + let platform_version = PlatformVersion::latest(); + let long = build_doc_type("note", note_schema(long_text())); + let unbounded = build_doc_type( + "note", + note_schema(platform_value!({"type": "string", "position": 0})), + ); + assert_eq!( + long.as_ref() + .max_size(platform_version) + .expect("the long string is sized"), + u16::MAX + ); + assert_eq!( + unbounded + .as_ref() + .max_size(platform_version) + .expect("the unbounded string is sized"), + u16::MAX + ); + + let platform_version_14 = PlatformVersion::get(14).expect("expected version 14"); + let long_14 = build_doc_type_at("note", note_schema(long_text()), platform_version_14); + assert!(matches!( + long_14.as_ref().max_size(platform_version_14), + Err(ProtocolError::Overflow(_)) + )); + } + + /// Below the cap both generations size every property alike, an object's + /// members and a typed array's elements included. + #[test] + fn should_size_bounded_properties_as_protocol_version_14_does() { + let platform_version_14 = PlatformVersion::get(14).expect("expected version 14"); + let schema = platform_value!({ + "type": "object", + "properties": { + "title": {"type": "string", "minLength": 3, "maxLength": 100, "position": 0}, + "count": {"type": "integer", "minimum": 0, "maximum": 1000, "position": 1}, + "data": {"type": "array", "byteArray": true, "maxItems": 64, "position": 2}, + "meta": { + "type": "object", + "properties": { + "name": {"type": "string", "maxLength": 10, "position": 0}, + "rank": {"type": "integer", "minimum": 0, "maximum": 9, "position": 1}, + }, + "additionalProperties": false, + "position": 3, + }, + "tags": { + "type": "array", + "items": {"type": "string", "maxLength": 10}, + "maxItems": 4, + "position": 4, + }, + }, + "additionalProperties": false, + }); + let at_latest = build_doc_type("doc", schema.clone()); + let at_14 = build_doc_type_at("doc", schema, platform_version_14); + let size = at_latest + .as_ref() + .max_size(PlatformVersion::latest()) + .expect("sized"); + assert_eq!( + size, + at_14.as_ref().max_size(platform_version_14).expect("sized") + ); + // The title alone is 400 bytes, and nothing comes near the cap + assert!(size > 400 && size < 1024, "{size}"); + } + // -------------------------------------------------------------- // DocumentTypeBasicMethods::requires_revision / initial_revision // -------------------------------------------------------------- diff --git a/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs b/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs index eba6769dfa4..f64e8af282a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs +++ b/packages/rs-dpp/src/data_contract/document_type/methods/versioned_methods.rs @@ -700,6 +700,27 @@ pub trait DocumentTypeV0MethodsVersioned: DocumentTypeV0Getters + DocumentTypeBa Ok(total_size) } + /// Generation 0 with each property sized by + /// [`DocumentPropertyType::saturating_max_byte_size`]: a string of 16384 + /// or more characters without `maxBytes` has a byte bound past `u16::MAX`, + /// which generation 0 fails on with an overflow and this generation holds + /// at `u16::MAX`. Every other type is sized as generation 0 sizes it, and + /// the total saturates as generation 0's does. + fn max_size_v1(&self, platform_version: &PlatformVersion) -> Result { + let mut total_size = 0u16; + + for document_property in self.flattened_properties().values() { + if let Some(size) = document_property + .property_type + .saturating_max_byte_size(platform_version)? + { + total_size = total_size.saturating_add(size); + } + } + + Ok(total_size) + } + /// Figures out the prefunded voting balance (v0) for a document in a document type fn prefunded_voting_balance_for_document_v0( &self, diff --git a/packages/rs-dpp/src/document/v0/serialize/header.rs b/packages/rs-dpp/src/document/v0/serialize/header.rs new file mode 100644 index 00000000000..2671f986c98 --- /dev/null +++ b/packages/rs-dpp/src/document/v0/serialize/header.rs @@ -0,0 +1,402 @@ +use crate::consensus::basic::decode::DecodingError; +use crate::data_contract::document_type::accessors::DocumentTypeV0Getters; +use crate::data_contract::document_type::methods::DocumentTypeBasicMethods; +use crate::data_contract::document_type::DocumentTypeRef; +use crate::data_contract::errors::DataContractError; +use crate::document::v0::DocumentV0; +use crate::nft::TradeMode; +use crate::prelude::Revision; +use crate::ProtocolError; +use integer_encoding::VarIntReader; +use platform_value::Identifier; +use platform_version::version::FeatureVersion; +use std::io::Read; + +impl DocumentV0 { + /// Reads the id and the revision of a serialized document from the fields + /// every serialization format (0 to 3) writes ahead of the document's + /// properties, without decoding the properties. + /// + /// The fields are the format, the contract version stamp (format 3), the + /// id, the owner id, the creator id (formats 2 and 3, for a type whose + /// documents can change hands) and the revision (for a type whose documents + /// carry one). Which of them are present depends on the type's + /// configuration, which a contract update cannot change, and never on its + /// properties. Bytes stored before an update changed how the properties + /// are encoded (the member order of a nested object, an optional member + /// added ahead of others) give their id and revision here, although + /// [`DocumentV0::from_bytes`] with the updated type no longer reads them. + /// For bytes [`DocumentV0::from_bytes`] reads, this returns the id and the + /// revision it decodes. + /// + /// The history migration on the first block of protocol version 15 keys + /// every retained revision by what this returns, so what it accepts and + /// returns for bytes written in formats 0 to 3 must never change. + pub fn id_and_revision_from_bytes( + mut serialized_document: &[u8], + document_type: DocumentTypeRef, + ) -> Result<(Identifier, Option), ProtocolError> { + let format: FeatureVersion = serialized_document + .read_varint() + .map_err(|error| header_error("serialization format", error))?; + match format { + 0..=2 => {} + 3 => { + let stamp: u64 = serialized_document + .read_varint() + .map_err(|error| header_error("contract version stamp", error))?; + if stamp > u32::MAX as u64 { + return Err(ProtocolError::DataContractError( + DataContractError::CorruptedSerialization( + "contract version stamp does not fit in a u32".to_string(), + ), + )); + } + } + version => { + return Err(ProtocolError::UnknownVersionMismatch { + method: "DocumentV0::id_and_revision_from_bytes".to_string(), + known_versions: vec![0, 1, 2, 3], + received: version, + }) + } + } + let mut id = [0; 32]; + serialized_document + .read_exact(&mut id) + .map_err(|error| header_error("id", error))?; + let mut identifier = [0; 32]; + serialized_document + .read_exact(&mut identifier) + .map_err(|error| header_error("owner id", error))?; + if format >= 2 + && (document_type.trade_mode() != TradeMode::None + || document_type.documents_transferable().is_transferable()) + { + let mut has_creator_id = [0]; + serialized_document + .read_exact(&mut has_creator_id) + .map_err(|error| header_error("creator id flag", error))?; + if has_creator_id[0] > 0 { + serialized_document + .read_exact(&mut identifier) + .map_err(|error| header_error("creator id", error))?; + } + } + let revision = if document_type.requires_revision() { + Some( + serialized_document + .read_varint() + .map_err(|error| header_error("revision", error))?, + ) + } else { + None + }; + Ok((Identifier::new(id), revision)) + } +} + +fn header_error(field: &str, error: std::io::Error) -> ProtocolError { + ProtocolError::DataContractError(DataContractError::DecodingDocumentError( + DecodingError::new(format!( + "error reading the {field} from a serialized document's header: {error}" + )), + )) +} + +#[cfg(test)] +mod tests { + use crate::data_contract::config::moderation::{ContractModerationConfig, ContractModerators}; + use crate::data_contract::config::DataContractConfig; + use crate::data_contract::document_type::accessors::DocumentTypeV0Getters; + use crate::data_contract::document_type::methods::DocumentTypeBasicMethods; + use crate::data_contract::document_type::DocumentType; + use crate::data_contract::errors::DataContractError; + use crate::document::serialization_traits::DocumentPlatformConversionMethodsV0; + use crate::document::v0::DocumentV0; + use crate::ProtocolError; + use integer_encoding::VarInt; + use platform_value::{platform_value, Identifier, Value}; + use platform_version::version::PlatformVersion; + use std::collections::BTreeMap; + + /// A type with an object property `o` holding `members` in this order and + /// an optional `status`, configured by `settings` (merged into the type's + /// schema), in a contract with moderators when `settings` gives them + /// abilities. + fn document_type(settings: Value, members: Vec<(&str, Value)>) -> DocumentType { + let platform_version = PlatformVersion::latest(); + let mut schema = platform_value!({ + "type": "object", + "properties": { + "o": { + "type": "object", "position": 0, + "properties": Value::Map( + members + .into_iter() + .map(|(name, schema)| (Value::Text(name.to_string()), schema)) + .collect(), + ), + "required": ["b", "c"], "additionalProperties": false, + }, + "status": {"type": "integer", "minimum": 0, "maximum": 3, "position": 1}, + }, + "required": ["o"], "additionalProperties": false, + }); + let Value::Map(settings) = settings else { + panic!("type settings are a map"); + }; + let mut moderated = false; + for (key, value) in settings { + let key = key.as_text().expect("a setting name"); + moderated |= key == "moderatorAbilities"; + schema.set_value(key, value).expect("the schema is a map"); + } + let config = + DataContractConfig::default_for_version(platform_version).expect("a default config"); + let config = if moderated { + config.with_moderation(Some(ContractModerationConfig { + banlist: false, + suspensions: false, + moderators: ContractModerators::ContractOwner, + warnings: false, + })) + } else { + config + }; + DocumentType::try_from_schema( + Identifier::new([2; 32]), + 1, + config.version(), + "note", + schema, + None, + &BTreeMap::new(), + &config, + false, + &mut Vec::new(), + platform_version, + ) + .expect("a valid document type") + } + + fn members(order: &[&'static str]) -> Vec<(&'static str, Value)> { + order + .iter() + .map(|name| { + let schema = match *name { + "a" => platform_value!({"type": "string", "maxLength": 20, "position": 2}), + "b" => platform_value!({"type": "string", "maxLength": 20, "position": 0}), + "c" => { + platform_value!({"type": "integer", "minimum": 0, "maximum": 10, "position": 1}) + } + other => panic!("no member {other}"), + }; + (*name, schema) + }) + .collect() + } + + fn document(revision: Option, creator_id: Option<[u8; 32]>) -> DocumentV0 { + let mut properties = BTreeMap::new(); + properties.insert("o".to_string(), platform_value!({"b": "hello", "c": 5u8})); + DocumentV0 { + id: Identifier::new([9; 32]), + owner_id: Identifier::new([7; 32]), + creator_id: creator_id.map(Identifier::new), + properties, + revision, + created_at: Some(1_000), + ..Default::default() + } + } + + /// Every type configuration that changes which header fields are + /// written: a revision or none, and a creator block or none. + fn configurations() -> Vec<(&'static str, Value)> { + vec![ + ("mutable", platform_value!({"documentsMutable": true})), + ("immutable", platform_value!({"documentsMutable": false})), + ( + "changed by its moderators", + platform_value!({ + "documentsMutable": false, + "moderatorAbilities": {"changeFields": ["status"]}, + }), + ), + ( + "transferable", + platform_value!({"documentsMutable": false, "transferable": 1}), + ), + ( + "sold by its owner", + platform_value!({"documentsMutable": false, "transferable": 1, "tradeMode": 1}), + ), + ] + } + + fn serialize(document: &DocumentV0, format: u16, document_type: &DocumentType) -> Vec { + let document_type = document_type.as_ref(); + match format { + 0 => document.serialize_v0(document_type), + 1 => document.serialize_v1(document_type), + 2 => document.serialize_v2(document_type), + 3 => document.serialize_v3(document_type), + _ => unreachable!("formats 0 to 3"), + } + .expect("a serializable document") + } + + #[test] + fn should_read_the_id_and_revision_from_bytes_decodes_in_every_format() { + let platform_version = PlatformVersion::latest(); + for (name, settings) in configurations() { + let document_type = document_type(settings, members(&["b", "c"])); + let moderated = document_type.as_ref().has_moderator_changeable_fields(); + if moderated { + assert!( + !document_type.as_ref().documents_mutable() + && !document_type + .as_ref() + .documents_transferable() + .is_transferable() + && document_type.as_ref().requires_revision(), + "the moderators' fields alone give the {name} type a revision" + ); + } + let revision = document_type.as_ref().requires_revision().then_some(4); + for creator_id in [None, Some([3; 32])] { + for format in 0..=3 { + let mut document = document(revision, creator_id); + if moderated && format == 3 { + // The moderation stamp sits in the time fields, after the + // revision; only format 3 has a place for it. + document.moderated_at = Some(2_000); + document.moderated_by = Some(Identifier::new([5; 32])); + } + let bytes = serialize(&document, format, &document_type); + let decoded = + DocumentV0::from_bytes(&bytes, document_type.as_ref(), platform_version) + .expect("bytes the type reads"); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes, document_type.as_ref()) + .expect("a readable header"), + (decoded.id, decoded.revision), + "{name} type, format {format}, creator {creator_id:?}" + ); + assert_eq!(decoded.revision, revision); + } + } + } + } + + #[test] + fn should_read_the_header_of_bytes_written_before_a_nested_member_change() { + let platform_version = PlatformVersion::latest(); + for (name, settings) in configurations() { + let written = document_type(settings.clone(), members(&["b", "c"])); + let revision = written.as_ref().requires_revision().then_some(4); + for (change, order) in [ + ("reordered", &["c", "b"][..]), + ("prepended optional member", &["a", "b", "c"][..]), + ] { + let updated = document_type(settings.clone(), members(order)); + for format in 0..=3 { + let bytes = serialize(&document(revision, Some([3; 32])), format, &written); + assert!( + DocumentV0::from_bytes(&bytes, updated.as_ref(), platform_version).is_err(), + "the {change} type no longer reads the {name} document in format {format}" + ); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes, updated.as_ref()) + .expect("a readable header"), + (Identifier::new([9; 32]), revision), + "{name} type, {change}, format {format}" + ); + } + } + } + } + + #[test] + fn should_refuse_a_header_it_cannot_read() { + let document_type = document_type( + platform_value!({"documentsMutable": true}), + members(&["b", "c"]), + ); + let bytes = serialize(&document(Some(4), None), 3, &document_type); + for truncated in [&bytes[..0], &bytes[..1], &bytes[..40], &bytes[..66]] { + assert!( + DocumentV0::id_and_revision_from_bytes(truncated, document_type.as_ref()).is_err(), + "{} bytes hold no complete header", + truncated.len() + ); + } + assert!(matches!( + DocumentV0::id_and_revision_from_bytes(&[4, 0], document_type.as_ref()), + Err(ProtocolError::UnknownVersionMismatch { received: 4, .. }) + )); + } + + #[test] + fn should_refuse_a_header_cut_inside_its_creator_id_block() { + let document_type = document_type( + platform_value!({"documentsMutable": false, "transferable": 1}), + members(&["b", "c"]), + ); + for format in [2, 3] { + let bytes = serialize(&document(Some(4), Some([3; 32])), format, &document_type); + // The format, an unset contract version stamp in format 3, the id + // and the owner id come before the creator id flag. + let flag = if format == 3 { 66 } else { 65 }; + assert_eq!(bytes[flag], 1, "format {format} marks a creator id"); + for (end, cut) in [ + (flag, "before the creator id flag"), + (flag + 1, "right after the creator id flag"), + (flag + 17, "inside the creator id"), + (flag + 33, "before the revision"), + ] { + assert!( + DocumentV0::id_and_revision_from_bytes(&bytes[..end], document_type.as_ref()) + .is_err(), + "format {format} cut {cut} holds no complete header" + ); + } + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&bytes[..flag + 34], document_type.as_ref()) + .expect("a complete header"), + (Identifier::new([9; 32]), Some(4)), + "format {format}" + ); + } + } + + #[test] + fn should_refuse_a_contract_version_stamp_the_full_reader_refuses() { + let platform_version = PlatformVersion::latest(); + let document_type = document_type( + platform_value!({"documentsMutable": true}), + members(&["b", "c"]), + ); + let bytes = serialize(&document(Some(4), None), 3, &document_type); + assert_eq!(bytes[1], 0, "an unset stamp is one byte"); + let with_stamp = |stamp: u64| [&bytes[..1], &stamp.encode_var_vec(), &bytes[2..]].concat(); + let largest = with_stamp(u32::MAX as u64); + assert!(DocumentV0::from_bytes(&largest, document_type.as_ref(), platform_version).is_ok()); + assert_eq!( + DocumentV0::id_and_revision_from_bytes(&largest, document_type.as_ref()) + .expect("a stamp that fits in a u32"), + (Identifier::new([9; 32]), Some(4)) + ); + let too_large = with_stamp(u32::MAX as u64 + 1); + assert!( + DocumentV0::from_bytes(&too_large, document_type.as_ref(), platform_version).is_err() + ); + assert!(matches!( + DocumentV0::id_and_revision_from_bytes(&too_large, document_type.as_ref()), + Err(ProtocolError::DataContractError( + DataContractError::CorruptedSerialization(_) + )) + )); + } +} diff --git a/packages/rs-dpp/src/document/v0/serialize/mod.rs b/packages/rs-dpp/src/document/v0/serialize/mod.rs index e9b41b7e24c..38f176a8bb9 100644 --- a/packages/rs-dpp/src/document/v0/serialize/mod.rs +++ b/packages/rs-dpp/src/document/v0/serialize/mod.rs @@ -25,6 +25,7 @@ use crate::consensus::ConsensusError; use crate::data_contract::accessors::v0::DataContractV0Getters; use crate::data_contract::config::DataContractConfig; +mod header; mod v0; mod v1; mod v2; diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs index 48171b1ac7c..81cfcc985ef 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/check_tx_verification/v0/mod.rs @@ -21,6 +21,7 @@ use crate::execution::types::state_transition_execution_context::{ StateTransitionExecutionContext, StateTransitionExecutionContextMethodsV0, }; use crate::execution::validation::state_transition::batch::choose_first_reservation; +use crate::execution::validation::state_transition::batch::erase_refund_recipient_reservation; use crate::execution::validation::state_transition::common::asset_lock::proof::verify_is_not_spent::AssetLockProofVerifyIsNotSpent; use crate::execution::validation::state_transition::processor::address_witnesses::{StateTransitionAddressWitnessValidationV0, StateTransitionHasAddressWitnessValidationV0}; use crate::execution::validation::state_transition::processor::traits::shielded_proof::{ @@ -533,14 +534,15 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } // A create choosing the first of the commitments to its values reads up to its scan's - // budget of competitors in a block; its state was not validated here, so the signer - // is held to that budget too + // budget of competitors in a block, and an erase is billed for the work its refunds + // cause; their state was not validated here, so the signer is held to both too if !validates_state { - reserve_choose_first_scans( + reserve_unvalidated_state_work( &action, + platform, &mut state_transition_execution_context, platform_version, - ); + )?; } let execution_event = ExecutionEvent::create_from_state_transition_action( @@ -741,13 +743,15 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } } - // As on the first check: the budget of a scan whose state was not validated + // As on the first check: the budget of a scan and the refund work of an erase + // whose state was not validated if !relies_on_gas_sponsor_to_pay { - reserve_choose_first_scans( + reserve_unvalidated_state_work( &action, + platform, &mut state_transition_execution_context, platform_version, - ); + )?; } let execution_event = ExecutionEvent::create_from_state_transition_action( @@ -771,18 +775,26 @@ pub(super) fn state_transition_to_execution_event_for_check_tx_v0<'a, C: CoreRPC } } -/// Reserves, on top of the estimated fee of `action`, what the `chooseFirst` checks of its -/// creates may bill in a block (see `choose_first_reservation`: their two hashes and the -/// competitor scan's budget), for an action whose state check tx did not validate. Fee +/// Reserves, on top of the estimated fee of `action`, what state validation bills in a block +/// for an action whose state check tx did not validate: what the `chooseFirst` checks of its +/// creates may bill (see `choose_first_reservation`: their two hashes and the competitor scan's +/// budget) and the work the refunds of its erases cause for other identities (see +/// `erase_refund_recipient_reservation`), priced at the epoch check tx estimates in. Fee /// validation then admits only a signer who could pay for them. -fn reserve_choose_first_scans( +fn reserve_unvalidated_state_work( action: &StateTransitionAction, + platform: &PlatformRef, execution_context: &mut StateTransitionExecutionContext, platform_version: &PlatformVersion, -) { - for operation in choose_first_reservation(action, platform_version) { - execution_context.add_operation(operation); - } +) -> Result<(), Error> { + execution_context.add_operations(choose_first_reservation(action, platform_version)); + execution_context.add_operations(erase_refund_recipient_reservation( + action, + platform.drive, + platform.state.last_committed_block_epoch_ref(), + platform_version, + )?); + Ok(()) } #[cfg(test)] diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs index a98788c0d28..774d8162e78 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/mod.rs @@ -14,6 +14,8 @@ use drive::state_transition_action::batch::batched_transition::document_transiti mod advanced_structure_v0; mod state_v0; +pub(in crate::execution::validation::state_transition) use state_v0::erase_refund_recipient_reservation; + pub trait DocumentEraseTransitionActionValidation { fn validate_structure( &self, diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs index 6285f88b77c..047214547fb 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/action_validation/document/document_erase_transition_action/state_v0/mod.rs @@ -1,3 +1,4 @@ +use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::execution_operation::ValidationOperation; use crate::execution::types::state_transition_execution_context::{ @@ -7,6 +8,7 @@ use crate::execution::validation::state_transition::batch::action_validation::do use crate::execution::validation::state_transition::batch::state::fetch_keep_history_document_lifecycle::fetch_keep_history_document_lifecycle; use crate::platform_types::platform::PlatformStateRef; use dpp::block::block_info::BlockInfo; +use dpp::block::epoch::Epoch; use dpp::consensus::basic::document::{ InvalidDocumentTransitionActionError, InvalidDocumentTypeError, }; @@ -20,7 +22,11 @@ use dpp::identifier::Identifier; use dpp::validation::SimpleConsensusValidationResult; use dpp::version::PlatformVersion; use drive::drive::document::lifecycle::DocumentLifecycleState; +use drive::drive::Drive; use drive::grovedb::TransactionArg; +use drive::state_transition_action::batch::batched_transition::document_transition::DocumentTransitionAction; +use drive::state_transition_action::batch::batched_transition::BatchedTransitionAction; +use drive::state_transition_action::StateTransitionAction; use drive::state_transition_action::batch::batched_transition::document_transition::document_base_transition_action::DocumentBaseTransitionActionAccessorsV0; use drive::state_transition_action::batch::batched_transition::document_transition::document_erase_transition_action::v0::DocumentEraseTransitionActionAccessorsV0; use drive::state_transition_action::batch::batched_transition::document_transition::document_erase_transition_action::DocumentEraseTransitionAction; @@ -141,3 +147,65 @@ impl DocumentEraseTransitionActionStateValidationV0 for DocumentEraseTransitionA } } } + +/// What check tx reserves, on top of the estimated fee of `action`, for the +/// erases of a batch whose state it does not validate: for every erase, the +/// work its refunds cause for other identities +/// ([`Drive::erase_refund_recipient_cost`]), which state validation 0 bills on +/// every path an erase takes. Nothing for anything else. Without it, a signer +/// covering an erase's estimated fee but not that work would be admitted, then +/// refused unpaid in a block, its nonce left to replay the work. A state +/// validation generation that bills an erase otherwise has to reserve what it +/// bills, so any other generation is refused here. +pub(in crate::execution::validation::state_transition) fn erase_refund_recipient_reservation( + action: &StateTransitionAction, + drive: &Drive, + epoch: &Epoch, + platform_version: &PlatformVersion, +) -> Result, Error> { + let StateTransitionAction::BatchAction(batch) = action else { + return Ok(Vec::new()); + }; + let erases = batch + .transitions() + .iter() + .filter(|transition| { + matches!( + transition, + BatchedTransitionAction::DocumentAction(DocumentTransitionAction::EraseAction(_)) + ) + }) + .count(); + if erases == 0 { + return Ok(Vec::new()); + } + match platform_version + .drive_abci + .validation_and_processing + .state_transitions + .batch_state_transition + .document_erase_transition_state_validation + { + Some(0) => {} + Some(version) => { + return Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "erase_refund_recipient_reservation".to_string(), + known_versions: vec![0], + received: version, + })) + } + None => { + return Err(Error::Execution(ExecutionError::VersionNotActive { + method: "erase_refund_recipient_reservation".to_string(), + known_versions: vec![0], + })) + } + } + let cost = drive + .erase_refund_recipient_cost(epoch, platform_version) + .map_err(Error::Drive)?; + Ok(vec![ + ValidationOperation::PrecalculatedOperation(cost); + erases + ]) +} diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs index 20d2fad4500..9a91f8daa04 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/mod.rs @@ -15,6 +15,9 @@ pub(in crate::execution::validation::state_transition) use state::v0::fetch_docu // Check tx reserves, for a create it does not validate the state of, what the block's // `chooseFirst` check may bill it: its two hashes and the competitor scan's budget. pub(in crate::execution::validation::state_transition) use action_validation::document::document_reference_validation::v0::choose_first_reservation; +// Check tx reserves, for an erase it does not validate the state of, the work its refunds cause +// for other identities, which the block's state validation bills. +pub(in crate::execution::validation::state_transition) use action_validation::document::document_erase_transition_action::erase_refund_recipient_reservation; // Moderator field changes and restores judge the document's `propertyConstraints`, with // the totals after the replace or insertion respectively. pub(in crate::execution::validation::state_transition) use transformer::v0::property_constraint_aggregates::{ diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs index 92f385774ab..5de5908a9be 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/erase.rs @@ -5,14 +5,21 @@ //! so an owner who loses their keys, their funds or their permission cannot //! strand a half-erased document. +use super::gas_sponsorship::gas_sponsorship_tests::IDENTITY_INSUFFICIENT_BALANCE; use super::*; +use crate::execution::check_tx::CheckTxLevel; +use crate::execution::check_tx::CheckTxLevel::{FirstTimeCheck, Recheck}; +use crate::platform_types::platform::PlatformRef; use crate::rpc::core::MockCoreRPCLike; use crate::test::helpers::setup::TempPlatform; +use dpp::block::epoch::Epoch; use dpp::consensus::basic::BasicError; +use dpp::consensus::codes::ErrorWithCode; use dpp::data_contract::document_type::accessors::DocumentTypeV2Getters; use dpp::data_contract::document_type::DocumentTypeRef; use dpp::data_contract::DataContract; use dpp::document::Document; +use dpp::fee::fee_result::FeeResult; use dpp::identifier::Identifier; use dpp::identity::{Identity, IdentityPublicKey, SecurityLevel}; use dpp::prelude::IdentityNonce; @@ -90,6 +97,12 @@ struct Fixture { impl Fixture { async fn new(document_type_name: &str) -> Self { + Self::new_with_message(document_type_name, None).await + } + + /// As [`Fixture::new`], with the document's `message` set to `message` + /// rather than a random one when given. + async fn new_with_message(document_type_name: &str, message: Option<&str>) -> Self { let platform_version = PlatformVersion::latest(); let mut platform = TestPlatformBuilder::new() .with_initial_protocol_version(platform_version.protocol_version) @@ -130,6 +143,9 @@ impl Fixture { platform_version, ) .expect("expected a random note"); + if let Some(message) = message { + document.set("message", message.into()); + } document .set_id_for_creation(document_type, &entropy.0, 1, platform_version) @@ -403,6 +419,61 @@ impl Fixture { .state } + /// The codes of the errors check tx at `level` refuses `transition` with, + /// and the fee it estimated when it got that far. + fn check_tx_at(&self, transition: &[u8], level: CheckTxLevel) -> (Vec, Option) { + let state = self.platform.state.load(); + let platform_ref = PlatformRef { + drive: &self.platform.drive, + state: &state, + config: &self.platform.config, + core_rpc: &self.platform.core_rpc, + }; + let result = self + .platform + .check_tx(transition, level, &platform_ref, PlatformVersion::latest()) + .expect("expected to check the transaction"); + let codes = result.errors.iter().map(|error| error.code()).collect(); + (codes, result.data.and_then(|check| check.fee_result)) + } + + /// Sets the owner's balance to `credits`. + fn set_owner_balance(&self, credits: Credits) { + let platform_version = PlatformVersion::latest(); + let id = self.owner.id().to_buffer(); + let balance = self + .platform + .drive + .fetch_identity_balance(id, None, platform_version) + .expect("expected to fetch the balance") + .expect("expected a balance"); + if balance > credits { + self.platform + .drive + .remove_from_identity_balance( + id, + balance - credits, + &BlockInfo::default(), + true, + None, + platform_version, + ) + .expect("expected to lower the balance"); + } else if credits > balance { + self.platform + .drive + .add_to_identity_balance( + id, + credits - balance, + &BlockInfo::default(), + true, + None, + platform_version, + ) + .expect("expected to raise the balance"); + } + } + fn lifecycle(&self, document_type_name: &str) -> DocumentLifecycleState { self.platform .drive @@ -882,8 +953,6 @@ async fn should_delete_and_erase_in_one_block() { /// in the estimate that admits the transition and in the fee actually charged. #[tokio::test] async fn should_charge_an_erase_for_the_refund_recipients_it_can_credit() { - use dpp::block::epoch::Epoch; - let mut fixture = Fixture::new("note").await; assert_successful(&fixture.delete_as_owner("note").await, "the delete"); @@ -1245,3 +1314,117 @@ async fn should_allow_the_new_owner_to_delete_and_erase_a_transferred_history() DocumentLifecycleState::Absent ); } + +/// Check tx validates no state of an unsponsored erase, and the work its +/// refunds cause for other identities is billed there, so check tx reserves +/// that work on top of the estimate, on the first check and on every recheck. +/// A signer one credit short of the estimate so reserved, who covers the +/// estimate without the reservation, stays out of the mempool rather than +/// being refused unpaid in a block; one holding the estimate is admitted. The +/// estimate the block admits the erase against is then the first check's and +/// the lifecycle read only state validation makes. +#[tokio::test] +async fn should_reserve_the_refund_recipient_work_when_admitting_an_erase() { + let mut fixture = Fixture::new("note").await; + assert_successful(&fixture.delete_as_owner("note").await, "the delete"); + let erase = fixture.erase_transition("note", true).await; + let recipient_cost = fixture + .platform + .drive + .erase_refund_recipient_cost(&Epoch::new(0).unwrap(), PlatformVersion::latest()) + .expect("expected the recipient work to be priced"); + + let mut first_check_estimate = None; + for level in [FirstTimeCheck, Recheck] { + fixture.set_owner_balance(dash_to_credits!(1.0)); + let (errors, estimated) = fixture.check_tx_at(&erase, level); + assert_eq!(errors, Vec::::new(), "{level:?}"); + let estimated = estimated.expect("expected an estimated fee"); + assert!( + estimated.processing_fee >= recipient_cost.processing_fee, + "{level:?}: the estimate holds the recipient work: {} is below {}", + estimated.processing_fee, + recipient_cost.processing_fee + ); + let required = estimated.total_base_fee(); + first_check_estimate.get_or_insert(required); + + fixture.set_owner_balance(required - 1); + assert_eq!( + fixture.check_tx_at(&erase, level).0, + vec![IDENTITY_INSUFFICIENT_BALANCE], + "{level:?}" + ); + fixture.set_owner_balance(required); + assert_eq!( + fixture.check_tx_at(&erase, level).0, + Vec::::new(), + "{level:?}" + ); + } + + let (_, lifecycle_read) = fixture + .platform + .drive + .fetch_document_lifecycle( + &fixture.contract, + fixture.document_type("note"), + fixture.document.id(), + Some(&Epoch::new(0).unwrap()), + None, + PlatformVersion::latest(), + ) + .expect("expected to read the lifecycle"); + fixture.set_owner_balance(dash_to_credits!(1.0)); + let result = process(&mut fixture.platform, erase); + let StateTransitionExecutionResult::SuccessfulExecution { estimated_fees, .. } = result else { + panic!("expected the erase to succeed, got {result:?}"); + }; + let block_estimate = estimated_fees + .expect("an erase is admitted against an estimate") + .total_base_fee(); + let first_check_estimate = first_check_estimate.expect("the first check estimated a fee"); + assert!( + first_check_estimate + lifecycle_read.total_base_fee() >= block_estimate, + "check tx admits the erase against all the block does but read its lifecycle: \ + {first_check_estimate} and a read of {} is below {block_estimate}", + lifecycle_read.total_base_fee() + ); +} + +/// A type whose string may hold 16384 characters and declares no `maxBytes` +/// has a byte bound past `u16::MAX` at four bytes a character. An erase sizes +/// the revisions it may remove as the type's largest documents, held at +/// `u16::MAX`, so it runs and is charged, and the estimate that admits it +/// covers a full chunk of revisions as large as a document's fields may be. +#[tokio::test] +async fn should_erase_a_type_whose_string_bound_does_not_fit_a_document_size() { + let platform_version = PlatformVersion::latest(); + let longest_message = "x".repeat(platform_version.system_limits.max_field_value_size as usize); + let mut fixture = Fixture::new_with_message("longNote", Some(&longest_message)).await; + let chunk = platform_version + .system_limits + .max_document_revisions_erased_per_transition + .expect("protocol 15 bounds the chunk") as u64; + fixture.retain_revisions("longNote", chunk); + assert_successful(&fixture.delete_as_owner("longNote").await, "the delete"); + let erase = fixture.erase_transition("longNote", true).await; + + let (errors, estimated) = fixture.check_tx_at(&erase, FirstTimeCheck); + assert_eq!(errors, Vec::::new()); + let estimated = estimated.expect("expected an estimated fee"); + let result = process(&mut fixture.platform, erase); + let StateTransitionExecutionResult::SuccessfulExecution { fee_result, .. } = result else { + panic!("expected the erase to succeed, got {result:?}"); + }; + assert!( + estimated.total_base_fee() >= fee_result.total_base_fee(), + "the estimate that admits the erase covers what it is charged: {} is below {}", + estimated.total_base_fee(), + fee_result.total_base_fee() + ); + assert_matches!( + fixture.lifecycle("longNote"), + DocumentLifecycleState::Absent + ); +} diff --git a/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json b/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json index 6c5816fe4ff..898ba7438e1 100644 --- a/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json +++ b/packages/rs-drive-abci/tests/supporting_files/contract/note/note-contract-keep-history-erasable.json @@ -61,6 +61,25 @@ "additionalProperties": false, "$comment": "A keep-history type whose deleted documents may also have their retained revisions purged. Deleting a note removes it from ordinary reads and leaves its revisions readable; erasing removes those revisions a bounded chunk at a time.", "transferable": 1 + }, + "longNote": { + "type": "object", + "documentsKeepHistory": true, + "documentsMutable": true, + "canBeDeleted": true, + "canBeErased": true, + "properties": { + "message": { + "type": "string", + "maxLength": 16384, + "position": 0 + } + }, + "required": [ + "message" + ], + "additionalProperties": false, + "$comment": "An erasable note whose message may hold 16384 characters and declares no maxBytes: at four bytes a character its byte bound does not fit in the u16 a document type's sizes are computed in." } }, "groups": {}, diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs index ce666d9eaf8..9f4280bc854 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v1/mod.rs @@ -190,8 +190,9 @@ impl Drive { DirectQueryType::StatelessDirectQuery { in_tree_type: TreeType::NormalTree, // A tree created at PV15 carries its creator's single-epoch owned flags; - // one migrated at activation carries the contract's flags, which can be larger. - // This is an average-case figure; the delete's total estimate covers the applied read. + // one migrated at activation keeps its legacy tree's flags, the creator's + // single-epoch owned flags or none. This is an average-case figure; the + // delete's total estimate covers the applied read. query_target: QueryTargetTree( StorageFlags::approximate_size(true, None), TreeType::ProvableCountTree, diff --git a/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs index 8060dd2afe6..6c38499511a 100644 --- a/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/erase_document_for_contract_operations/v0/mod.rs @@ -94,14 +94,13 @@ impl Drive { contract.id_ref().as_bytes(), document_type.name().as_str(), ); - // Worst-case sizing: a chunk may hold the type's largest documents, and - // the admission estimate has to cover that whatever this document - // actually stores. - let revision_size = document_type.max_size(platform_version)? as u32; - let mut batch_operations: Vec = vec![]; if let Some(layers) = estimated_costs_only_with_layer_info { + // Worst-case sizing: a chunk may hold the type's largest documents, + // and the admission estimate has to cover that whatever this + // document actually stores. + let revision_size = document_type.max_size(platform_version)? as u32; Self::add_estimation_costs_for_erase_document( document_id, contract, diff --git a/packages/rs-drive/src/drive/document/lifecycle/tests.rs b/packages/rs-drive/src/drive/document/lifecycle/tests.rs index d3945440d4d..99f099f4744 100644 --- a/packages/rs-drive/src/drive/document/lifecycle/tests.rs +++ b/packages/rs-drive/src/drive/document/lifecycle/tests.rs @@ -3221,6 +3221,9 @@ fn should_cover_applied_delete_fees_for_migrated_history_with_multi_epoch_contra let drive = setup_drive_with_initial_state_structure(None); let owner = [7; 32]; let mut schema = json_document_to_json_value(FAMILY_HISTORY_CONTRACT).unwrap(); + // A deletable contract, so the legacy writer flags the document's history + // tree with the document's own flags. + schema["config"] = serde_json::json!({"$formatVersion": "0", "canBeDeleted": true}); let contract = DataContract::from_json(schema.clone(), false, old).unwrap(); drive .apply_contract( @@ -3300,13 +3303,13 @@ fn should_cover_applied_delete_fees_for_migrated_history_with_multi_epoch_contra ) .value .unwrap(); - let flags = StorageFlags::map_cow_some_element_flags_ref(tree.get_flags()) - .unwrap() - .unwrap(); - assert!(matches!(flags.as_ref(), StorageFlags::MultiEpochOwned(..))); - assert!( - tree.get_flags().as_ref().unwrap().len() - > StorageFlags::approximate_size(true, None) as usize + assert_eq!( + tree.get_flags(), + &StorageFlags::map_to_some_element_flags(Some(&StorageFlags::new_single_epoch( + 0, + Some(owner) + ))), + "the migrated history tree keeps its legacy tree's flags, not the contract's" ); assert_delete_estimate_covers_applied( &drive, diff --git a/packages/rs-drive/src/drive/document/migration/index_tests.rs b/packages/rs-drive/src/drive/document/migration/index_tests.rs index 32b1aa91ce0..ac6c08e4ae4 100644 --- a/packages/rs-drive/src/drive/document/migration/index_tests.rs +++ b/packages/rs-drive/src/drive/document/migration/index_tests.rs @@ -13,7 +13,7 @@ use dpp::block::block_info::BlockInfo; use dpp::block::epoch::Epoch; use dpp::data_contract::DataContractFactory; use dpp::document::document_factory::DocumentFactory; -use dpp::document::DocumentV0Setters; +use dpp::document::{Document, DocumentV0Getters, DocumentV0Setters}; use dpp::platform_value::{platform_value, Value}; use std::borrow::Cow; diff --git a/packages/rs-drive/src/drive/document/migration/mod.rs b/packages/rs-drive/src/drive/document/migration/mod.rs index 2b70d443f5b..260f585ee57 100644 --- a/packages/rs-drive/src/drive/document/migration/mod.rs +++ b/packages/rs-drive/src/drive/document/migration/mod.rs @@ -15,8 +15,7 @@ use crate::util::storage_flags::StorageFlags; use dpp::data_contract::accessors::v0::DataContractV0Getters; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dpp::data_contract::document_type::DocumentPropertyType; -use dpp::document::serialization_traits::DocumentPlatformConversionMethodsV0; -use dpp::document::{Document, DocumentV0Getters}; +use dpp::document::DocumentV0; use dpp::version::PlatformVersion; use grovedb::batch::{QualifiedGroveDbOp, SubelementsDeletionBehavior}; use grovedb::reference_path::ReferencePathType::{SiblingReference, UpstreamRootHeightReference}; @@ -40,7 +39,7 @@ pub struct DocumentHistoryMigrationStats { pub maximum_document_revisions: u64, /// Document index references inspected. pub index_entries: u64, - /// Index references successfully rewritten. + /// Index references rewritten. pub rewritten_index_entries: u64, /// Largest index-reference count retained for one document type. pub maximum_type_index_entries: u64, @@ -55,6 +54,24 @@ pub struct DocumentHistoryMigrationStats { type IndexReference = (Vec>, Vec, Element); type IndexEntries = BTreeMap, Vec>; +/// Writes gathered across documents, types and contracts and applied together, so the root is +/// recomputed once per batch rather than once per document. +#[derive(Default)] +struct PendingHistoryWrites { + /// History trees and their revisions to insert, and the legacy per-document trees to delete. + moves: Vec, + /// Current pointers written over the deleted trees' keys, and the index references that now + /// target them. A batch can not delete a key's tree and write a reference at that key, or + /// point a reference into a tree it deletes, so these are applied after `moves`. + links: Vec, +} + +impl PendingHistoryWrites { + fn len(&self) -> usize { + self.moves.len() + self.links.len() + } +} + #[cfg(test)] mod index_tests; #[cfg(test)] @@ -99,6 +116,12 @@ fn index_inventory_buffer_bound(entries: &IndexEntries) -> u64 { impl Drive { /// Moves timestamp-keyed document histories out of primary storage. /// The caller owns the transaction and must roll it back on any error. + /// + /// Retained revisions are copied unchanged. Only their id and revision are + /// read, from the fields the serialized document holds ahead of its + /// properties, so a revision is moved whatever the type's current schema + /// makes of its properties. The writes are applied in batches of about + /// `max_history_migration_batch_operations` operations. pub fn migrate_document_history_storage( &self, transaction: &Transaction, @@ -109,6 +132,14 @@ impl Drive { "document history migration requires the composite-key layout", )); } + let max_batch_operations = platform_version + .system_limits + .max_history_migration_batch_operations + .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( + "the protocol version that migrates document histories sets \ + max_history_migration_batch_operations", + )))? as usize; + let mut pending = PendingHistoryWrites::default(); let mut stats = DocumentHistoryMigrationStats::default(); let mut cursor: Option<[u8; 32]> = None; loop { @@ -183,12 +214,14 @@ impl Drive { .iter() .any(|(key, _)| key == &[DOCUMENT_HISTORY_TREE_KEY]) { - self.history_migration_batch( - vec![QualifiedGroveDbOp::insert_or_replace_op( - type_path.clone(), - vec![DOCUMENT_HISTORY_TREE_KEY], - Element::empty_tree_with_flags(contract_flags.clone()), - )], + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( + type_path.clone(), + vec![DOCUMENT_HISTORY_TREE_KEY], + Element::empty_tree_with_flags(contract_flags.clone()), + )); + self.history_migration_apply_when_full( + &mut pending, + max_batch_operations, transaction, &mut stats, platform_version, @@ -235,11 +268,17 @@ impl Drive { )?; let mut pointer = None; let mut revisions = BTreeMap::new(); - let mut copies = vec![QualifiedGroveDbOp::insert_or_replace_op( + // The history tree keeps the flags of the per-document tree it + // replaces, which the legacy writer flagged with the document's + // own flags when the contract can be deleted, and left unflagged + // otherwise. + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( history_path[..history_path.len() - 1].to_vec(), document_id.clone(), - Element::empty_provable_count_tree_with_flags(contract_flags.clone()), - )]; + Element::empty_provable_count_tree_with_flags( + element.get_flags().clone(), + ), + )); for (key, element) in &entries { if key == &[0] { pointer = Some(element.clone()); @@ -251,12 +290,18 @@ impl Drive { let Element::Item(bytes, _) = element else { return Err(corrupt("legacy history revision must be an item")); }; - let document = Document::from_bytes( + stats.revisions += 1; + stats.revision_bytes += bytes.len() as u64; + let (id, revision) = DocumentV0::id_and_revision_from_bytes( bytes, document_type.as_ref(), - platform_version, - )?; - if document.id().as_slice() != document_id { + ) + .map_err(|error| { + corrupt(format!( + "legacy history revision has no readable header: {error}" + )) + })?; + if id.as_slice() != document_id { return Err(corrupt( "history document id does not match its primary key", )); @@ -264,97 +309,70 @@ impl Drive { let time = DocumentPropertyType::decode_date_timestamp(key) .ok_or_else(|| corrupt("invalid history timestamp"))?; let mut composite = encode_u64(time); - composite.extend(encode_u64(document.revision().unwrap_or(1))); + composite.extend(encode_u64(revision.unwrap_or(1))); revisions.insert(key.clone(), composite.clone()); - copies.push(QualifiedGroveDbOp::insert_or_replace_op( + pending.moves.push(QualifiedGroveDbOp::insert_or_replace_op( history_path.clone(), composite, element.clone(), )); } - let revision_entries = entries - .iter() - .filter(|(key, _)| key != &[0]) - .cloned() - .collect::>(); - Self::history_migration_count_revisions(&revision_entries, &mut stats)?; + stats.maximum_document_revisions = + stats.maximum_document_revisions.max(revisions.len() as u64); let pointer = Self::history_migration_pointer( pointer.ok_or_else(|| corrupt("history has no current pointer"))?, &document_id, &revisions, )?; - self.history_migration_batch( - copies, - transaction, - &mut stats, - platform_version, - )?; - let mut deletes = entries - .into_iter() - .map(|(key, _)| QualifiedGroveDbOp::delete_op(old_path.clone(), key)) - .collect::>(); - deletes.push(QualifiedGroveDbOp::delete_tree_op( + pending.moves.extend( + entries.into_iter().map(|(key, _)| { + QualifiedGroveDbOp::delete_op(old_path.clone(), key) + }), + ); + pending.moves.push(QualifiedGroveDbOp::delete_tree_op( primary_path.clone(), document_id.clone(), old_tree_type, SubelementsDeletionBehavior::Error, )); - self.history_migration_batch( - deletes, - transaction, - &mut stats, - platform_version, - )?; - self.history_migration_batch( - vec![QualifiedGroveDbOp::insert_or_replace_op( - primary_path.clone(), - document_id.clone(), - pointer, - )], + pending.links.push(QualifiedGroveDbOp::insert_or_replace_op( + primary_path.clone(), + document_id.clone(), + pointer, + )); + for (path, key, mut element) in + index_entries.remove(&document_id).unwrap_or_default() + { + match &mut element { + Element::Reference(reference, hops, _) + | Element::ReferenceWithSumItem(reference, hops, _, _) => { + *reference = UpstreamRootHeightReference( + 4, + vec![vec![0], document_id.clone()], + ); + *hops = Some(2); + } + // The inventory only admits references; anything + // else is corrupt state, reported like the rest. + _ => { + return Err(corrupt( + "unexpected leaf in a historical document index", + )) + } + } + pending + .links + .push(QualifiedGroveDbOp::insert_or_replace_op(path, key, element)); + stats.rewritten_index_entries += 1; + } + stats.migrated_documents += 1; + self.history_migration_apply_when_full( + &mut pending, + max_batch_operations, transaction, &mut stats, platform_version, )?; - if let Some(references) = index_entries.remove(&document_id) { - let rewrites = references - .iter() - .cloned() - .map(|(path, key, mut element)| { - match &mut element { - Element::Reference(reference, hops, _) - | Element::ReferenceWithSumItem(reference, hops, _, _) => { - *reference = UpstreamRootHeightReference( - 4, - vec![vec![0], document_id.clone()], - ); - *hops = Some(2); - } - // The inventory only admits references; anything - // else is corrupt state, reported like the rest. - _ => { - return Err(corrupt( - "unexpected leaf in a historical document index", - )) - } - } - Ok(QualifiedGroveDbOp::insert_or_replace_op(path, key, element)) - }) - .collect::, Error>>()?; - self.history_migration_batch( - rewrites, - transaction, - &mut stats, - platform_version, - )?; - self.history_migration_check_index_rewrites( - &references, - &document_id, - transaction, - &mut stats, - platform_version, - )?; - } - stats.migrated_documents += 1; } if !index_entries.is_empty() { return Err(corrupt("history index references a missing document")); @@ -362,48 +380,50 @@ impl Drive { } } } + self.history_migration_apply(&mut pending, transaction, &mut stats, platform_version)?; Ok(stats) } - fn history_migration_check_index_rewrites( + /// Applies the pending writes once they hold `max_batch_operations` operations. + fn history_migration_apply_when_full( &self, - references: &[IndexReference], - document_id: &[u8], + pending: &mut PendingHistoryWrites, + max_batch_operations: usize, transaction: &Transaction, stats: &mut DocumentHistoryMigrationStats, platform_version: &PlatformVersion, ) -> Result<(), Error> { - let mut rewritten = 0; - for (path, key, original) in references { - let mut expected = original.clone(); - match &mut expected { - Element::Reference(reference, hops, _) - | Element::ReferenceWithSumItem(reference, hops, _, _) => { - *reference = - UpstreamRootHeightReference(4, vec![vec![0], document_id.to_vec()]); - *hops = Some(2); - } - _ => continue, + if pending.len() >= max_batch_operations { + self.history_migration_apply(pending, transaction, stats, platform_version)?; + } + Ok(()) + } + + /// Applies the pending moves, then the pending links. + fn history_migration_apply( + &self, + pending: &mut PendingHistoryWrites, + transaction: &Transaction, + stats: &mut DocumentHistoryMigrationStats, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + for operations in [ + std::mem::take(&mut pending.moves), + std::mem::take(&mut pending.links), + ] { + if operations.is_empty() { + continue; } - let stored = self - .grove - .get_raw( - path.as_slice().into(), - key, + self.grove + .apply_batch( + operations, + None, Some(transaction), &platform_version.drive.grove_version, ) .value?; - if stored == expected { - rewritten += 1; - } + stats.batches += 1; } - if rewritten != references.len() { - return Err(corrupt( - "history index rewrite count differs from its inventory", - )); - } - stats.rewritten_index_entries += rewritten as u64; Ok(()) } @@ -535,23 +555,4 @@ impl Drive { } Ok(()) } - - fn history_migration_batch( - &self, - operations: Vec, - transaction: &Transaction, - stats: &mut DocumentHistoryMigrationStats, - platform_version: &PlatformVersion, - ) -> Result<(), Error> { - self.grove - .apply_batch( - operations, - None, - Some(transaction), - &platform_version.drive.grove_version, - ) - .value?; - stats.batches += 1; - Ok(()) - } } diff --git a/packages/rs-drive/src/drive/document/migration/tests.rs b/packages/rs-drive/src/drive/document/migration/tests.rs index 300dbc4554a..cf66bf8f795 100644 --- a/packages/rs-drive/src/drive/document/migration/tests.rs +++ b/packages/rs-drive/src/drive/document/migration/tests.rs @@ -1,10 +1,25 @@ use super::*; +use crate::drive::document::paths::document_history_path; +use crate::drive::RootTree; use crate::query::{SingleDocumentDriveQuery, SingleDocumentDriveQueryContestedStatus}; use crate::util::object_size_info::{DocumentAndContractInfo, DocumentInfo, OwnedDocumentInfo}; use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; use dpp::block::block_info::BlockInfo; -use dpp::document::DocumentV0Setters; -use dpp::tests::json_document::{json_document_to_contract, json_document_to_document}; +use dpp::data_contract::accessors::v0::DataContractV0Setters; +use dpp::data_contract::config::v0::DataContractConfigGettersV0; +use dpp::data_contract::conversion::json::DataContractJsonConversionMethodsV0; +use dpp::data_contract::validate_update::DataContractUpdateValidationMethodsV0; +use dpp::data_contract::{DataContract, DataContractFactory}; +use dpp::document::document_factory::DocumentFactory; +use dpp::document::serialization_traits::DocumentPlatformConversionMethodsV0; +use dpp::document::{Document, DocumentV0Getters, DocumentV0Setters}; +use dpp::platform_value::{platform_value, Value}; +use dpp::tests::json_document::{ + json_document_to_contract, json_document_to_document, json_document_to_json_value, +}; +use grovedb::AggregateData; +use grovedb_merk::element::reconstruct::ElementReconstructExtensions; +use std::borrow::Cow; #[test] fn should_accept_history_migration_when_only_primary_writer_changes() { @@ -164,6 +179,16 @@ fn migrate_summable_history(revision_count: u64) -> DocumentHistoryMigrationStat let old_entries = drive .history_migration_entries(&old_path, &transaction, old) .unwrap(); + let legacy_tree = drive + .grove + .get_raw( + (&old_path[..old_path.len() - 1]).into(), + document.id().as_slice(), + Some(&transaction), + &old.drive.grove_version, + ) + .value + .unwrap(); let stats = drive .migrate_document_history_storage(&transaction, new) .unwrap(); @@ -244,8 +269,13 @@ fn migrate_summable_history(revision_count: u64) -> DocumentHistoryMigrationStat .unwrap(); assert_eq!( history_tree.get_flags(), - &StorageFlags::map_to_some_element_flags(Some(&contract_flags)), - "new structural bytes inherit the contract's flags" + legacy_tree.get_flags(), + "the history tree keeps the flags of the per-document tree it replaces" + ); + assert_eq!( + legacy_tree.get_flags(), + &None, + "the legacy writer leaves a history tree unflagged when the contract can not be deleted" ); let history_root = drive .grove @@ -352,7 +382,10 @@ fn should_migrate_revisions_and_indexes_without_recovering_overwritten_revisions assert_eq!(stats.revisions, 2); assert_eq!(stats.migrated_documents, 1); assert!(stats.index_entries > 0); - assert_eq!(stats.rewritten_index_entries, stats.index_entries); + assert_eq!( + assert_index_references_rewritten(&drive, &contract, "profile", &transaction), + stats.index_entries + ); let root_migrated = drive .grove .root_hash(Some(&transaction), &new.drive.grove_version) @@ -556,92 +589,6 @@ fn should_reject_unrecognised_type_children_during_migration_inventory() { } } -#[test] -fn should_halt_migration_when_an_inventoried_index_reference_was_not_rewritten() { - let old = PlatformVersion::get(14).unwrap(); - let new = PlatformVersion::get(15).unwrap(); - let drive = setup_drive_with_initial_state_structure(Some(old)); - let contract = json_document_to_contract( - "tests/supporting_files/contract/dashpay/dashpay-contract-with-profile-history.json", - false, - old, - ) - .unwrap(); - drive - .apply_contract(&contract, BlockInfo::default(), true, None, None, old) - .unwrap(); - let document_type = contract.document_type_for_name("profile").unwrap(); - let document = json_document_to_document( - "tests/supporting_files/contract/dashpay/profile0.json", - Some([7; 32].into()), - document_type, - old, - ) - .unwrap(); - drive - .add_document_for_contract( - DocumentAndContractInfo { - owned_document_info: OwnedDocumentInfo { - document_info: DocumentInfo::DocumentRefInfo((&document, None)), - owner_id: None, - }, - contract: &contract, - document_type, - }, - true, - BlockInfo::default_with_time(1000), - true, - None, - old, - ) - .unwrap(); - let transaction = drive.grove.start_transaction(); - let type_path = contract_document_type_path_vec(contract.id().as_slice(), "profile"); - let mut stats = DocumentHistoryMigrationStats::default(); - let mut inventory = IndexEntries::new(); - for (key, _) in drive - .history_migration_entries(&type_path, &transaction, old) - .unwrap() - { - if key == [0] { - continue; - } - let mut path = type_path.clone(); - path.push(key); - drive - .history_migration_index_entries(path, &transaction, &mut stats, &mut inventory, old) - .unwrap(); - } - let references = inventory.remove(document.id().as_slice()).unwrap(); - assert!(!references.is_empty()); - let error = drive - .history_migration_check_index_rewrites( - &references, - document.id().as_slice(), - &transaction, - &mut stats, - new, - ) - .expect_err("an omitted rewrite must halt instead of being counted as rewritten"); - assert!(error - .to_string() - .contains("history index rewrite count differs from its inventory")); - assert_eq!(stats.rewritten_index_entries, 0); - drive - .migrate_document_history_storage(&transaction, new) - .unwrap(); - drive - .history_migration_check_index_rewrites( - &references, - document.id().as_slice(), - &transaction, - &mut stats, - new, - ) - .unwrap(); - assert_eq!(stats.rewritten_index_entries, references.len() as u64); -} - /// A keep-history type with a contested index could be registered before /// protocol 15. Activation loads every stored contract through the structural /// parse, which must keep reading that combination, or the migration would @@ -727,3 +674,754 @@ fn should_migrate_a_legacy_contested_keep_history_contract() { assert_eq!(stats.types, 1); assert_eq!(stats.migrated_documents, 1); } + +/// Writes `document` of type `name` as protocol 14 does, in the block at +/// `time_ms`, with `flags`. +fn add_legacy_revision( + drive: &Drive, + contract: &DataContract, + name: &str, + document: &Document, + flags: StorageFlags, + time_ms: u64, +) { + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentInfo::DocumentRefInfo(( + document, + Some(Cow::Owned(flags)), + )), + owner_id: None, + }, + contract, + document_type: contract.document_type_for_name(name).unwrap(), + }, + true, + BlockInfo::default_with_time(time_ms), + true, + None, + PlatformVersion::get(14).unwrap(), + ) + .unwrap(); +} + +/// Checks that every index reference of type `name` targets its document's +/// current pointer, two hops from the revision it resolves to, and returns +/// how many there are. +fn assert_index_references_rewritten( + drive: &Drive, + contract: &DataContract, + name: &str, + transaction: &Transaction, +) -> u64 { + let version = PlatformVersion::latest(); + let type_path = contract_document_type_path_vec(contract.id().as_slice(), name); + let mut stats = DocumentHistoryMigrationStats::default(); + let mut inventory = IndexEntries::new(); + for index in contract + .document_type_for_name(name) + .unwrap() + .index_structure() + .sub_levels() + .keys() + { + let mut path = type_path.clone(); + path.push(index.as_bytes().to_vec()); + drive + .history_migration_index_entries(path, transaction, &mut stats, &mut inventory, version) + .unwrap(); + } + for (document_id, references) in inventory { + let current_pointer = vec![vec![0], document_id]; + for (path, key, element) in references { + assert!( + matches!( + &element, + Element::Reference(UpstreamRootHeightReference(4, target), Some(2), _) + | Element::ReferenceWithSumItem( + UpstreamRootHeightReference(4, target), + Some(2), + _, + _ + ) if target == ¤t_pointer + ), + "index reference {path:?}/{key:?} was not rewritten: {element:?}" + ); + } + } + stats.index_entries +} + +/// A keep-history type `note` whose property `o` is an object holding +/// `members`, in this order, in contract version `version`. +fn nested_member_contract(members: Vec<(&str, Value)>, version: u32) -> DataContract { + let members = Value::Map( + members + .into_iter() + .map(|(name, schema)| (Value::Text(name.to_string()), schema)) + .collect(), + ); + let schema = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": true, + "canBeDeleted": false, + "properties": { + "o": { + "type": "object", "position": 0, "properties": members, + "required": ["b", "c"], "additionalProperties": false, + }, + }, + "required": ["o"], "additionalProperties": false, + }); + let mut contract = DataContractFactory::new(14) + .unwrap() + .create_with_value_config( + [7; 32].into(), + 0, + platform_value!({"note": schema}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + contract.set_version(version); + contract +} + +/// Protocol 14 accepts a contract update that reorders a nested object's +/// members or prepends an optional one, and properties are encoded in member +/// order. Revisions stored before such an update keep their bytes, so the +/// migration moves them without reading their properties. +#[test] +fn should_migrate_revisions_stored_before_a_nested_property_change() { + let old = PlatformVersion::get(14).unwrap(); + let new = PlatformVersion::latest(); + let a = platform_value!({"type": "string", "maxLength": 20, "position": 2}); + let b = platform_value!({"type": "string", "maxLength": 20, "position": 0}); + let c = platform_value!({"type": "integer", "minimum": 0, "maximum": 10, "position": 1}); + let original = nested_member_contract(vec![("b", b.clone()), ("c", c.clone())], 1); + for (change, updated) in [ + ( + "reordered members", + nested_member_contract(vec![("c", c.clone()), ("b", b.clone())], 2), + ), + ( + "prepended optional member", + nested_member_contract( + vec![("a", a.clone()), ("b", b.clone()), ("c", c.clone())], + 2, + ), + ), + ] { + assert!( + original + .validate_update(&updated, &BlockInfo::default(), old) + .unwrap() + .is_valid(), + "protocol 14 accepts the update with {change}" + ); + let drive = setup_drive_with_initial_state_structure(Some(old)); + drive + .apply_contract(&original, BlockInfo::default(), true, None, None, old) + .unwrap(); + let mut document = DocumentFactory::new(14) + .unwrap() + .create_document( + &original, + [7; 32].into(), + "note".into(), + platform_value!({"o": {"b": "hello", "c": 5}}), + ) + .unwrap(); + document.set_id([9; 32].into()); + for revision in 1..=2u64 { + document.set_revision(Some(revision)); + document.set("o", platform_value!({"b": "hello", "c": revision + 3})); + add_legacy_revision( + &drive, + &original, + "note", + &document, + StorageFlags::new_single_epoch(revision as u16, Some([7; 32])), + revision * 1000, + ); + } + drive + .apply_contract( + &updated, + BlockInfo::default_with_time(5000), + true, + None, + None, + old, + ) + .unwrap(); + + let transaction = drive.grove.start_transaction(); + let mut legacy_path = contract_document_type_path_vec(original.id().as_slice(), "note"); + legacy_path.extend([vec![0], document.id().to_vec()]); + let legacy = drive + .history_migration_entries(&legacy_path, &transaction, old) + .unwrap() + .into_iter() + .filter(|(key, _)| key != &[0]) + .collect::>(); + assert_eq!(legacy.len(), 2); + let Element::Item(stored, _) = &legacy[0].1 else { + panic!("a retained revision is an item"); + }; + assert!( + Document::from_bytes(stored, updated.document_type_for_name("note").unwrap(), new) + .is_err(), + "the fixture must hold revisions the updated type no longer reads ({change})" + ); + + let stats = drive + .migrate_document_history_storage(&transaction, new) + .unwrap_or_else(|error| panic!("migration after {change} failed: {error}")); + assert_eq!(stats.migrated_documents, 1); + let migrated = drive + .history_migration_entries( + &document_history_path(original.id().as_slice(), "note", document.id().as_slice()), + &transaction, + new, + ) + .unwrap(); + let expected = legacy + .into_iter() + .zip(1u64..) + .map(|((time, element), revision)| { + let mut key = + encode_u64(DocumentPropertyType::decode_date_timestamp(&time).unwrap()); + key.extend(encode_u64(revision)); + (key, element) + }) + .collect::>(); + assert_eq!( + migrated, expected, + "every revision keeps its bytes and flags under its time and revision ({change})" + ); + } +} + +/// The legacy writer gave each document's history tree the document's own +/// flags when the contract can be deleted. The migrated tree keeps them, so +/// erasing the document's history credits its owner, not the contract's owner. +#[test] +fn should_keep_the_owner_flags_on_the_migrated_history_tree() { + let old = PlatformVersion::get(14).unwrap(); + let new = PlatformVersion::latest(); + let contract_owner = [66; 32]; + let document_owner = [14; 32]; + // A deletable keep-history type is refused at registration since protocol + // 14, but stored contracts from before hold them, so the stored contract is + // read without validation. + let mut schema = json_document_to_json_value( + "tests/supporting_files/contract/family/family-contract-with-history.json", + ) + .unwrap(); + schema["config"] = serde_json::json!({"$formatVersion": "0", "canBeDeleted": true}); + let contract = DataContract::from_json(schema, false, old).unwrap(); + assert!(contract.config().can_be_deleted()); + let drive = setup_drive_with_initial_state_structure(Some(old)); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + Some(Cow::Owned(StorageFlags::new_single_epoch( + 0, + Some(contract_owner), + ))), + None, + old, + ) + .unwrap(); + let document_type = contract.document_type_for_name("person").unwrap(); + let mut document = json_document_to_document( + "tests/supporting_files/contract/family/person0.json", + Some(document_owner.into()), + document_type, + old, + ) + .unwrap(); + let owner_flags = StorageFlags::new_single_epoch(0, Some(document_owner)); + for revision in 1..=2u64 { + document.set_revision(Some(revision)); + add_legacy_revision( + &drive, + &contract, + "person", + &document, + owner_flags.clone(), + revision * 1000, + ); + } + let type_path = contract_document_type_path_vec(contract.id().as_slice(), "person"); + let mut primary_path = type_path.clone(); + primary_path.push(vec![0]); + let legacy_tree = drive + .grove + .get_raw( + primary_path.as_slice().into(), + document.id().as_slice(), + None, + &old.drive.grove_version, + ) + .value + .unwrap(); + assert_eq!( + legacy_tree.get_flags(), + &StorageFlags::map_to_some_element_flags(Some(&owner_flags)), + "the legacy writer flags a deletable contract's history tree with its document's flags" + ); + + let transaction = drive.grove.start_transaction(); + drive + .migrate_document_history_storage(&transaction, new) + .unwrap(); + drive + .grove + .commit_transaction(transaction) + .unwrap() + .unwrap(); + let mut history_root = type_path.clone(); + history_root.push(vec![DOCUMENT_HISTORY_TREE_KEY]); + let history_tree = drive + .grove + .get_raw( + history_root.as_slice().into(), + document.id().as_slice(), + None, + &new.drive.grove_version, + ) + .value + .unwrap(); + assert!(matches!(history_tree, Element::ProvableCountTree(..))); + assert_eq!(history_tree.get_flags(), legacy_tree.get_flags()); + + let mut operations = vec![]; + let delete = drive + .delete_document_for_contract_operations( + document.id(), + &contract, + document_type, + &BlockInfo::default_with_time(5_000), + Some(document_owner.into()), + None, + &mut None, + None, + new, + ) + .unwrap(); + drive + .apply_batch_low_level_drive_operations(None, None, delete, &mut operations, &new.drive) + .unwrap(); + let erase_block = BlockInfo::default_with_time(6_000); + let erase = drive + .erase_document_for_contract_operations( + document.id(), + &contract, + document_type, + &erase_block, + &mut None, + None, + new, + ) + .unwrap(); + let mut operations = vec![]; + drive + .apply_batch_low_level_drive_operations(None, None, erase, &mut operations, &new.drive) + .unwrap(); + let refunds = drive + .calculate_fee(None, Some(operations), &erase_block.epoch, new) + .unwrap() + .fee_refunds; + assert!( + drive + .grove + .get_raw( + history_root.as_slice().into(), + document.id().as_slice(), + None, + &new.drive.grove_version, + ) + .value + .is_err(), + "the erase removes the history tree" + ); + assert!(refunds.0.contains_key(&document_owner)); + assert!( + !refunds.0.contains_key(&contract_owner), + "nothing the erase removes was paid for by the contract's owner: {refunds:?}" + ); +} + +/// Two contracts holding legacy histories of every shape the migration moves: +/// a summable type, an indexed type with several revisions (two of them +/// written in one block, so one is overwritten), an immutable type, and a type +/// with no documents. +fn legacy_mixed_histories() -> (Drive, [DataContract; 2]) { + let old = PlatformVersion::get(14).unwrap(); + let drive = setup_drive_with_initial_state_structure(Some(old)); + let note = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": true, + "canBeDeleted": false, + "properties": { + "label": {"type": "string", "maxLength": 32, "position": 0}, + "rank": {"type": "integer", "minimum": 0, "maximum": 1000, "position": 1}, + }, + "required": ["label", "rank"], "additionalProperties": false, + "indices": [ + {"name": "byLabel", "properties": [{"label": "asc"}]}, + {"name": "byRank", "properties": [{"rank": "asc"}]}, + ], + }); + let tip = platform_value!({ + "type": "object", "documentsKeepHistory": true, "canBeDeleted": false, + "documentsCountable": true, "documentsSummable": "amount", + "properties": {"amount": {"type": "integer", "minimum": 0, "maximum": 4294967295i64, "position": 0}}, + "required": ["amount"], "additionalProperties": false, + "indices": [{"name": "amount", "properties": [{"amount": "asc"}]}], + }); + let stamp = platform_value!({ + "type": "object", "documentsKeepHistory": true, "documentsMutable": false, + "canBeDeleted": false, + "properties": {"label": {"type": "string", "maxLength": 32, "position": 0}}, + "required": ["label"], "additionalProperties": false, + "indices": [{"name": "byLabel", "properties": [{"label": "asc"}]}], + }); + let factory = DataContractFactory::new(14).unwrap(); + let first = factory + .create_with_value_config( + [7; 32].into(), + 0, + platform_value!({"note": note.clone(), "tip": tip, "stamp": stamp, "empty": note.clone()}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + let second = factory + .create_with_value_config( + [8; 32].into(), + 1, + platform_value!({"note": note}), + None, + None, + ) + .unwrap() + .data_contract_owned(); + for (contract, epoch) in [(&first, 1), (&second, 2)] { + drive + .apply_contract( + contract, + BlockInfo::default(), + true, + Some(Cow::Owned(StorageFlags::new_single_epoch( + epoch, + Some(contract.owner_id().to_buffer()), + ))), + None, + old, + ) + .unwrap(); + } + // (contract, type, id byte, each write's block time and values) + let histories = vec![ + ( + &first, + "note", + 1, + vec![(1_000, platform_value!({"label": "a", "rank": 1}))], + ), + ( + &first, + "note", + 2, + vec![ + (1_000, platform_value!({"label": "b", "rank": 2})), + (2_000, platform_value!({"label": "b", "rank": 3})), + (2_000, platform_value!({"label": "b2", "rank": 4})), + (3_000, platform_value!({"label": "b3", "rank": 5})), + ], + ), + ( + &first, + "note", + 3, + vec![(1_500, platform_value!({"label": "c", "rank": 2}))], + ), + ( + &first, + "tip", + 4, + vec![ + (1_000, platform_value!({"amount": 5})), + (4_000, platform_value!({"amount": 7})), + ], + ), + ( + &first, + "tip", + 5, + vec![(1_000, platform_value!({"amount": 11}))], + ), + ( + &first, + "stamp", + 6, + vec![(2_500, platform_value!({"label": "s"}))], + ), + ( + &second, + "note", + 7, + vec![ + (1_000, platform_value!({"label": "x", "rank": 9})), + (5_000, platform_value!({"label": "y", "rank": 9})), + ], + ), + ( + &second, + "note", + 8, + vec![(1_000, platform_value!({"label": "z", "rank": 0}))], + ), + ]; + for (contract, name, id, writes) in histories { + let mut document = DocumentFactory::new(14) + .unwrap() + .create_document( + contract, + contract.owner_id(), + name.into(), + writes[0].1.clone(), + ) + .unwrap(); + document.set_id([id; 32].into()); + for (index, (time, value)) in writes.into_iter().enumerate() { + if document.revision().is_some() { + document.set_revision(Some(index as u64 + 1)); + } + if let Value::Map(fields) = value { + for (field, value) in fields { + document.set(field.as_text().unwrap(), value); + } + } + add_legacy_revision( + &drive, + contract, + name, + &document, + StorageFlags::new_single_epoch((time / 1_000) as u16, Some([id + 100; 32])), + time, + ); + } + } + (drive, [first, second]) +} + +/// Every element under `path`, by its path below `path`, with each tree's +/// root key cleared and its count and sum kept: the root key records the +/// shape of the tree under it, which depends on how its writes were grouped +/// into batches. +fn stored_elements( + drive: &Drive, + path: &[Vec], + transaction: &Transaction, +) -> BTreeMap>, Element> { + let version = PlatformVersion::latest(); + let mut elements = BTreeMap::new(); + for (key, element) in drive + .history_migration_entries(path, transaction, version) + .unwrap() + { + if !element.is_any_tree() { + elements.insert(vec![key], element); + continue; + } + let mut child = path.to_vec(); + child.push(key.clone()); + for (below, element) in stored_elements(drive, &child, transaction) { + elements.insert([vec![key.clone()], below].concat(), element); + } + let aggregate = AggregateData::CountAndSum( + element.count_value_or_default(), + element.sum_value_or_default(), + ); + elements.insert( + vec![key], + element.reconstruct_with_root_key(None, aggregate).unwrap(), + ); + } + elements +} + +/// Migrates in batches of `max_batch_operations` operations, commits, and +/// returns every element the contracts' documents tree holds afterwards. +fn migrated_elements( + drive: &Drive, + contracts: &[DataContract], + max_batch_operations: u32, +) -> ( + BTreeMap>, Element>, + DocumentHistoryMigrationStats, +) { + let mut version = PlatformVersion::latest().clone(); + version.system_limits.max_history_migration_batch_operations = Some(max_batch_operations); + let transaction = drive.grove.start_transaction(); + let stats = drive + .migrate_document_history_storage(&transaction, &version) + .unwrap(); + drive + .grove + .commit_transaction(transaction) + .unwrap() + .unwrap(); + let issues = drive + .grove + .verify_grovedb(None, true, false, &version.drive.grove_version) + .unwrap(); + assert!( + issues.is_empty(), + "batches of {max_batch_operations} operations leave inconsistent hashes: {issues:?}" + ); + let transaction = drive.grove.start_transaction(); + let mut index_entries = 0; + for contract in contracts { + for name in contract.document_types().keys() { + index_entries += assert_index_references_rewritten(drive, contract, name, &transaction); + } + } + assert_eq!(index_entries, stats.rewritten_index_entries); + let elements = stored_elements( + drive, + &[vec![RootTree::DataContractDocuments as u8]], + &transaction, + ); + (elements, stats) +} + +/// The batch size decides only how the writes are grouped, so every size +/// leaves the same elements, and GroveDB's own hashes agree with them, also +/// when a batch ends inside a type or between contracts, holds several +/// documents, or holds less than one document's writes. +#[test] +fn should_store_the_same_histories_whatever_the_batch_size() { + let default = PlatformVersion::latest() + .system_limits + .max_history_migration_batch_operations + .unwrap(); + let (drive, contracts) = legacy_mixed_histories(); + let (expected, expected_stats) = migrated_elements(&drive, &contracts, 1); + assert_eq!(expected_stats.migrated_documents, 8); + assert_eq!(expected_stats.revisions, 12); + assert_eq!(expected_stats.rewritten_index_entries, 13); + assert_eq!( + expected_stats.batches, + 5 + 8 * 2, + "one batch per history tree a type lacks, then per document its moves and its links" + ); + // Every document here writes at least seven operations, so up to seven a + // batch holds one document; from sixteen a batch holds several and can end + // inside a type; from seventy-four one pair of batches holds everything. + for (max_batch_operations, batches) in [ + (2, 17), + (7, 16), + (16, 8), + (20, 6), + (default, 2), + (u32::MAX, 2), + ] { + let (drive, contracts) = legacy_mixed_histories(); + let (elements, stats) = migrated_elements(&drive, &contracts, max_batch_operations); + assert_eq!( + elements, expected, + "batches of {max_batch_operations} operations" + ); + assert_eq!( + stats.batches, batches, + "batches of {max_batch_operations} operations" + ); + assert_eq!(stats.migrated_documents, expected_stats.migrated_documents); + assert_eq!(stats.revisions, expected_stats.revisions); + assert_eq!( + stats.rewritten_index_entries, + expected_stats.rewritten_index_entries + ); + } +} + +/// A migration that stops after some batches were applied leaves nothing +/// once its transaction is dropped, and a run in a new transaction stores +/// what an uninterrupted one does. +#[test] +fn should_store_the_same_histories_after_a_run_that_stopped_part_way() { + let (drive, contracts) = legacy_mixed_histories(); + let (reference, _) = { + let (drive, contracts) = legacy_mixed_histories(); + migrated_elements(&drive, &contracts, 1) + }; + let mut version = PlatformVersion::latest().clone(); + version.system_limits.max_history_migration_batch_operations = Some(1); + let last = contracts + .iter() + .max_by_key(|contract| contract.id()) + .unwrap(); + let last_type = contract_document_type_path_vec(last.id().as_slice(), "note"); + drive + .grove + .insert( + last_type.as_slice(), + b"unknown", + Element::empty_tree(), + None, + None, + &version.drive.grove_version, + ) + .unwrap() + .unwrap(); + let root_before = drive + .grove + .root_hash(None, &version.drive.grove_version) + .unwrap() + .unwrap(); + let transaction = drive.grove.start_transaction(); + let stopped = drive.migrate_document_history_storage(&transaction, &version); + assert!(stopped.is_err()); + assert_ne!( + drive + .grove + .root_hash(Some(&transaction), &version.drive.grove_version) + .unwrap() + .unwrap(), + root_before, + "the first contract's documents were written before the run stopped" + ); + drop(transaction); + assert_eq!( + drive + .grove + .root_hash(None, &version.drive.grove_version) + .unwrap() + .unwrap(), + root_before, + "nothing the stopped run wrote outlives its transaction" + ); + drive + .grove + .delete( + last_type.as_slice(), + b"unknown", + None, + None, + &version.drive.grove_version, + ) + .unwrap() + .unwrap(); + let (elements, stats) = migrated_elements(&drive, &contracts, 1); + assert_eq!(stats.migrated_documents, 8); + assert_eq!(elements, reference); +} diff --git a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs index c58b4b21b62..170d1b18ee3 100644 --- a/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs +++ b/packages/rs-platform-version/src/version/dpp_versions/dpp_contract_versions/v7.rs @@ -46,7 +46,10 @@ use crate::version::dpp_versions::dpp_contract_versions::{ /// format version 1; /// - `canonical_contested_index_values`, `Some(1)` rather than `Some(0)`: every index value /// of a contest is written the one way its property's type keys it, so the values of one -/// contest name one poll; 0 wrote only identifiers one way. +/// contest name one poll; 0 wrote only identifiers one way; +/// - `max_size`, 1 rather than 0: a string of 16384 or more characters without `maxBytes` +/// counts as `u16::MAX` bytes toward a document type's largest size rather than failing it +/// with an overflow. Only the erase of retained revisions, new in this version, reads it. /// /// v6 remains unchanged for `PROTOCOL_VERSION_14`, whose parse refuses a byte array key and /// whose schemas and documents the `jsonschema` crate checks. @@ -91,6 +94,8 @@ pub const CONTRACT_VERSIONS_V7: DPPContractVersions = DPPContractVersions { // changed: every index value of a contest is written the one way its property's // type keys it, where 0 wrote only identifiers one way canonical_contested_index_values: Some(1), + // changed: a string bound past `u16::MAX` bytes is held there, not an overflow + max_size: 1, ..CONTRACT_VERSIONS_V6.document_type_versions.methods }, ..CONTRACT_VERSIONS_V6.document_type_versions diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 19ff20f0c9f..60dd8951291 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -682,6 +682,7 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, + max_history_migration_batch_operations: None, }, consensus: ConsensusVersions { tenderdash_consensus_version: 0, diff --git a/packages/rs-platform-version/src/version/system_limits/mod.rs b/packages/rs-platform-version/src/version/system_limits/mod.rs index c572525e802..5bf4c9ae048 100644 --- a/packages/rs-platform-version/src/version/system_limits/mod.rs +++ b/packages/rs-platform-version/src/version/system_limits/mod.rs @@ -568,6 +568,22 @@ pub struct SystemLimits { /// /// `None` in the tables of protocol versions that never run the rewrite. pub max_rekey_batch_moves: Option, + /// How many GroveDB operations the move of retained document histories to the composite-key + /// layout on the first block of protocol version 15 + /// (`Drive::migrate_document_history_storage`) gathers before it applies them, across + /// documents, types and contracts: one batch per document would propagate to the root once + /// per document. A document's operations are never split, so a batch holds at most one + /// document's operations more. Part of what that block writes, since the batch boundaries + /// shape the Merk trees the documents' keys are deleted from and inserted into. + /// + /// It also bounds the block's stack: a batch deletes the keys of up to about a sixth of + /// this many documents from a type's primary-key tree, and Merk's batch delete recurses + /// about once per key it removes from a tree. 500 operations stay under 512 KiB of stack + /// for a type of 32,000 documents; 10,000 need more than 2 MiB. + /// + /// `None` in the tables of protocol versions before 15, which never run the migration; every + /// table from protocol version 15 on sets it. + pub max_history_migration_batch_operations: Option, } #[cfg(test)] diff --git a/packages/rs-platform-version/src/version/system_limits/v1.rs b/packages/rs-platform-version/src/version/system_limits/v1.rs index d4c2f8c9b3b..f635693f797 100644 --- a/packages/rs-platform-version/src/version/system_limits/v1.rs +++ b/packages/rs-platform-version/src/version/system_limits/v1.rs @@ -116,4 +116,5 @@ pub const SYSTEM_LIMITS_V1: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v2.rs b/packages/rs-platform-version/src/version/system_limits/v2.rs index 9a74de90d71..0b936126554 100644 --- a/packages/rs-platform-version/src/version/system_limits/v2.rs +++ b/packages/rs-platform-version/src/version/system_limits/v2.rs @@ -97,4 +97,5 @@ pub const SYSTEM_LIMITS_V2: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v3.rs b/packages/rs-platform-version/src/version/system_limits/v3.rs index da29418d9b7..9d4b394de1b 100644 --- a/packages/rs-platform-version/src/version/system_limits/v3.rs +++ b/packages/rs-platform-version/src/version/system_limits/v3.rs @@ -99,4 +99,5 @@ pub const SYSTEM_LIMITS_V3: SystemLimits = SystemLimits { max_index_entries_per_document: None, max_choose_first_candidates: 1_000, // read only by document reference validation 0 at v14 max_rekey_batch_moves: None, // read only by the protocol version 14 upgrade + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v4.rs b/packages/rs-platform-version/src/version/system_limits/v4.rs index 6ef0d5f2efa..19eb8eb7200 100644 --- a/packages/rs-platform-version/src/version/system_limits/v4.rs +++ b/packages/rs-platform-version/src/version/system_limits/v4.rs @@ -220,4 +220,5 @@ pub const SYSTEM_LIMITS_V4: SystemLimits = SystemLimits { max_index_entries_per_document: Some(256), // typed array element indexes (new in v14): one document of a type with an indexed array writes at most 256 index entries max_choose_first_candidates: 1_000, // refersTo chooseFirst (new in v14): competitors a create reads, oldest first max_rekey_batch_moves: Some(10_000), // the protocol version 14 rewrite of unsigned index value keys applies its moves in batches of this many moves + max_history_migration_batch_operations: None, // read only by the protocol version 15 upgrade }; diff --git a/packages/rs-platform-version/src/version/system_limits/v5.rs b/packages/rs-platform-version/src/version/system_limits/v5.rs index ffe2e8d18e0..6ff2322cb40 100644 --- a/packages/rs-platform-version/src/version/system_limits/v5.rs +++ b/packages/rs-platform-version/src/version/system_limits/v5.rs @@ -22,6 +22,9 @@ use crate::version::system_limits::SystemLimits; /// * Retained-history erasure: one transition removes at most 100 revisions /// (`max_document_revisions_erased_per_transition`, `None` before), as enforced by /// `erase_document_for_contract_operations` 0. +/// * Document history layout: the first block of the version moves retained document histories +/// to the composite-key layout in batches of about 500 GroveDB operations +/// (`max_history_migration_batch_operations`, `None` before). pub const SYSTEM_LIMITS_V5: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: Some(100), // one bounded lifecycle erasure chunk (note 41) max_compiled_contract_size: Some(1_048_576), // changed: contracts stored compiled take at most 1 MiB @@ -30,5 +33,6 @@ pub const SYSTEM_LIMITS_V5: SystemLimits = SystemLimits { private_document_store_min_chunk_power: Some(4), // changed: and its chunk power is at least 4 private_document_store_max_chunk_power: Some(11), // changed: and at most 11 max_private_document_entries_per_transition: Some(16), // changed: a PrivateDocumentCreateFromShieldedPool appends at most 16 entries + max_history_migration_batch_operations: Some(500), // changed: the protocol version 15 move of retained document histories applies its writes in batches of about this many operations ..SYSTEM_LIMITS_V4 }; diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index 2a3c212d2a9..5bca17022e7 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -892,7 +892,10 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// deletion time, and an erasing or erased document is absent at every requested time. /// Parser 4 admits `canBeErased` with meta-schema 3; compiled type storage keeps generation /// 0 and its existing non-lifecycle bytes. Protocols through 14 retain their handlers, -/// limits and wire bounds, and cannot admit an erase. +/// limits and wire bounds, and cannot admit an erase. An erase sizes the revisions it may +/// remove with `max_size` 1 (`CONTRACT_VERSIONS_V7`), which counts a string of 16384 or +/// more characters without `maxBytes` as `u16::MAX` bytes where generation 0 fails on the +/// overflow. /// 42. **A contract's patterns are bounded**: `CONTRACT_VERSIONS_V7` selects /// `validate_schema_patterns` 0. Contract registration and update refuse a contract whose /// document schemas and `$defs` hold more than 32 patterns @@ -931,7 +934,7 @@ pub const PLATFORM_V15: PlatformVersion = PlatformVersion { state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V2, state_transitions: STATE_TRANSITION_VERSIONS_V5, // changed: validate_asset_lock_transaction_structure 1 drops the input cap and validate_instant_asset_lock_proof_structure 1 applies it to instant proofs only, so a chain proof redeems a lock of more than 100 inputs (note 30) - contract_versions: CONTRACT_VERSIONS_V7, // changed: parse_property_constraints 1 lets countOf and sumOf match byte array keys (note 4); validate_document 2 checks documents natively (note 6); check_document_type_schema 1 checks schemas natively at registration (note 7); contract_storage_serialization_version 2 stores contracts compiled (note 8); parse_private_document 0 parses private document types (note 13); apply_generated_from 1 admits sys.tokens.decimals (note 15); try_from_schema 4 parses lifecycle flags (note 41); validate_schema_patterns 0 bounds the patterns of a contract and registration_cost 3 estimates its fee without compiling one (note 42) + contract_versions: CONTRACT_VERSIONS_V7, // changed: parse_property_constraints 1 lets countOf and sumOf match byte array keys (note 4); validate_document 2 checks documents natively (note 6); check_document_type_schema 1 checks schemas natively at registration (note 7); contract_storage_serialization_version 2 stores contracts compiled (note 8); parse_private_document 0 parses private document types (note 13); apply_generated_from 1 admits sys.tokens.decimals (note 15); try_from_schema 4 parses lifecycle flags (note 41); max_size 1 holds a string bound past u16::MAX bytes there for the erase (note 41); validate_schema_patterns 0 bounds the patterns of a contract and registration_cost 3 estimates its fee without compiling one (note 42) document_versions: DOCUMENT_VERSIONS_V4, identity_versions: IDENTITY_VERSIONS_V1, voting_versions: VOTING_VERSION_V3, // changed: the yes/no poll's stored info, id, parameter rules and decision rules are active (note 10), and the decision proposal's poll (note 12) @@ -942,7 +945,7 @@ pub const PLATFORM_V15: PlatformVersion = PlatformVersion { }, system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V4, // changed: DPNS v3 checks a name's create with keywords (note 3); the decision proposals contract (note 12); the token market contract is selected (note 17); DashPay v3 states what its data triggers checked (note 24) fee_version: FEE_VERSION4, // changed: decision proposals prefund their poll with 0.5 Dash (note 12); a private document entry pays its storage and 10,000,000 credits (note 25); a multiple choice proposal pays a 0.2 Dash processing fee on create (note 40) - system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41) + system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41); max_history_migration_batch_operations batches the move of retained document histories consensus: ConsensusVersions { tenderdash_consensus_version: 1, evidence: Some(ConsensusEvidenceParams {