diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/mod.rs index ff51957cedd..a1194675243 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/mod.rs @@ -92,11 +92,11 @@ pub use keyword::{ ObjectKeyword, RelationshipKeyword, ResolutionKeyword, SchemaKeyword, StorageKeyword, StringKeyword, }; -pub use validate::DocumentSchemaValidator; /// Registration's native schema checks compare values and read list uniqueness as the /// document validator does. #[cfg(feature = "validation")] pub(crate) use validate::{equal_json, is_unique}; +pub use validate::{DocumentSchemaValidator, EnumViolationListing}; /// A JSON Schema simple type. #[derive( diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/tests.rs b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/tests.rs index 9350d8729aa..b6bdc8828da 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/tests.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/tests.rs @@ -376,7 +376,17 @@ pub(super) fn generated_schema( insert("multipleOf", cases.pick(&divisors)); } if cases.chance(10) { - insert("enum", json!([cases.pick(values), cases.pick(values)])); + // Now and then an enum long enough to be looked up rather than scanned + let members: Vec = if cases.chance(30) { + values + .iter() + .filter(|_| cases.chance(70)) + .cloned() + .collect() + } else { + vec![cases.pick(values), cases.pick(values)] + }; + insert("enum", serde_json::Value::Array(members)); } if cases.chance(8) { insert("const", cases.pick(values)); diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/mod.rs index c5a64db3e11..d58b55716de 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/mod.rs @@ -21,6 +21,19 @@ use pattern::PatternCell; #[cfg(feature = "validation")] pub(crate) use v0::{equal_json, is_unique}; +/// How much of an `enum`'s allowed values a violation lists, in its parameters and its +/// message: the longest run of them, from the first the schema writes, of at most +/// `max_values` values, each `null`, a boolean, a number or a string, whose JSON takes at most +/// `max_bytes` bytes together. The others are counted, not listed, so the error stays small +/// whatever the enum holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EnumViolationListing { + /// The most allowed values listed. + pub max_values: usize, + /// The most bytes the listed values' JSON takes together. + pub max_bytes: usize, +} + /// A document type's validation schema ready to check document values: each of its patterns /// checked to parse, and compiled the first time a value reaches it, and the generation of /// `validation_schema` that built it, which checks them. The schema is shared, so a clone, or @@ -103,14 +116,16 @@ impl DocumentSchemaValidator { /// validator's depth check gives a too-deep value. That check, which runs before the /// walk, is what bounds a document's own value depth; this bound guards the walk's /// recursion for a caller that reaches it another way. The generation that built the - /// validator checks it. + /// validator checks it. An `enum` violation lists the first `max_enum_listed_values` of + /// the allowed values and counts the others (`None` lists them all). pub fn validate( &self, instance: &JsonValue, max_value_depth: Option, + enum_listing: Option, ) -> Result, ProtocolError> { match self.generation { - 0 => self.validate_v0(instance, max_value_depth), + 0 => self.validate_v0(instance, max_value_depth, enum_listing), version => Err(ProtocolError::UnknownVersionMismatch { method: "DocumentSchemaValidator::validate".to_string(), known_versions: vec![0], diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/mod.rs b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/mod.rs index 4e883f4ed13..bc3b760f896 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/mod.rs @@ -32,7 +32,7 @@ use super::super::{ NumberConstraints, ObjectConstraints, SchemaNode, StringConstraints, ValidationSchema, ValidatorFormat, }; -use super::{DocumentSchemaValidator, PatternCell}; +use super::{DocumentSchemaValidator, EnumViolationListing, PatternCell}; use crate::consensus::basic::json_schema_error::JsonSchemaError; use crate::consensus::basic::value_error::ValueError; use crate::consensus::basic::BasicError; @@ -47,7 +47,7 @@ use regex::Regex; use serde_json::{json, Map as JsonMap, Number as JsonNumber, Value as JsonValue}; use std::cell::RefCell; use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::net::IpAddr; @@ -65,6 +65,11 @@ mod tests; /// JSON value instead. const UNIQUE_ITEMS_PAIRWISE_LIMIT: usize = 15; +/// The `enum` length up to which the walk compares a value with each allowed value in turn; +/// it looks a value up among a longer enum's values by [`MemberKey`] instead, which reaches +/// the same verdict. +const ENUM_SCAN_LIMIT: usize = 16; + /// A date's digits, as the JSON Schema validator requires them besides the date parse, /// which ignores missing zero padding. static DATE: Lazy = @@ -202,8 +207,10 @@ impl DocumentSchemaValidator { &self, instance: &JsonValue, max_value_depth: Option, + enum_listing: Option, ) -> Result, ProtocolError> { let mut walk = Walk::new(self, false, max_value_depth); + walk.enum_listing = enum_listing; walk.node(&self.schema.root, instance, 0, 0)?; if let Some(depth) = *walk.too_deep.borrow() { // Guarded only when a limit is set, so one is @@ -260,6 +267,14 @@ struct Walk<'v, 'i> { /// The value-nesting depth at which the walk first went past `max_value_depth`, shared /// with the walks a `contains` spawns so one going too deep refuses the whole value. too_deep: Rc>>, + /// The allowed values of each `enum` longer than [`ENUM_SCAN_LIMIT`] the walk has met, by + /// the address of the schema's list, as the keys it looks a value up by: built the first + /// time the walk checks a value against the list, and shared with the walks a `contains` + /// spawns, so each value costs one lookup however many values the list holds. Each list's + /// keys are held with the deepest nesting any of them has. + enum_members: Rc>>, + /// How much of an `enum`'s allowed values a violation lists, or `None` for all of them. + enum_listing: Option, /// Ties the walk to the checked value's lifetime `'i`, so the borrow checker refuses a /// value that does not outlive the walk: every address in `held` stays valid while it is /// held, and a temporary can never be checked against the cache. @@ -291,6 +306,8 @@ impl<'v, 'i> Walk<'v, 'i> { held: Rc::new(RefCell::new(HashSet::new())), max_value_depth, too_deep: Rc::new(RefCell::new(None)), + enum_members: Rc::new(RefCell::new(HashMap::new())), + enum_listing: None, instance: PhantomData, } } @@ -310,6 +327,8 @@ impl<'v, 'i> Walk<'v, 'i> { held: Rc::clone(&self.held), max_value_depth: self.max_value_depth, too_deep: Rc::clone(&self.too_deep), + enum_members: Rc::clone(&self.enum_members), + enum_listing: None, instance: PhantomData, } } @@ -477,10 +496,16 @@ impl<'v, 'i> Walk<'v, 'i> { return Ok(()); } if let Some(values) = &node.enum_values { - if !values.iter().any(|value| equal_value(instance, value)) { + if !self.enum_admits(values, instance) { + let listing = self.enum_listing; self.fail("enum", "", || { - let options = json_of_values(values); - let message = format!("{instance} is not one of {options}"); + let (options, unlisted) = listed_values(values, listing); + let message = match unlisted { + 0 => format!("{instance} is not one of {options}"), + unlisted => { + format!("{instance} is not one of {options} or {unlisted} more") + } + }; (json!({ "enum": options }), message) }); } @@ -546,6 +571,23 @@ impl<'v, 'i> Walk<'v, 'i> { Ok(()) } + /// Whether `instance` equals one of `values`, an `enum`'s allowed values, as + /// [`equal_value`] compares them: by a scan of a short list, and by a lookup of the + /// value's [`MemberKey`] among a longer list's, whose keys are built once per walk. + fn enum_admits(&self, values: &'v [Value], instance: &JsonValue) -> bool { + if values.len() <= ENUM_SCAN_LIMIT { + return values.iter().any(|value| equal_value(instance, value)); + } + let mut members = self.enum_members.borrow_mut(); + let (keys, depth) = members.entry(values.as_ptr() as usize).or_insert_with(|| { + let keys: HashSet = values.iter().filter_map(MemberKey::of_value).collect(); + let depth = keys.iter().map(MemberKey::depth).max().unwrap_or(0); + (keys, depth) + }); + // A value nested deeper than every allowed value equals none, and is not read further + MemberKey::of_json(instance, *depth).is_some_and(|key| keys.contains(&key)) + } + fn object( &mut self, object: &'v ObjectConstraints, @@ -1286,3 +1328,132 @@ fn json_of_value(value: &Value) -> JsonValue { fn json_of_values(values: &[Value]) -> JsonValue { JsonValue::Array(values.iter().map(json_of_value).collect()) } + +/// `values`, an `enum`'s allowed values, as JSON for a violation to list: those `listing` +/// admits, the longest run from the first that holds no array or object and fits its count +/// and its bytes, or all of them when there is no listing, and how many are left out. +fn listed_values(values: &[Value], listing: Option) -> (JsonValue, usize) { + let Some(listing) = listing else { + return (json_of_values(values), 0); + }; + let mut listed = Vec::new(); + let mut bytes = 0; + for value in values.iter().take(listing.max_values) { + let json = json_of_value(value); + if json.is_array() || json.is_object() { + break; + } + bytes += json.to_string().len(); + if bytes > listing.max_bytes { + break; + } + listed.push(json); + } + let unlisted = values.len() - listed.len(); + (JsonValue::Array(listed), unlisted) +} + +/// The keys of an `enum`'s allowed values, with the deepest nesting any of them has. +type EnumMembers = (HashSet, usize); + +/// A value as [`equal_value`] compares it with an `enum`'s allowed value: two values are +/// equal exactly when their keys are. A number is keyed by its value whatever its reading +/// (an integral float as the integer it is), an array element by element, and an object +/// entry by entry in the order it holds them, as [`equal_value`] pairs them. +#[derive(PartialEq, Eq, Hash)] +enum MemberKey { + Null, + Bool(bool), + /// A number with an integral value within `i128`, whatever its reading. + Integer(i128), + /// Any other number, by its f64 bits. + Float(u64), + Text(String), + Array(Vec), + Object(Vec<(String, MemberKey)>), +} + +impl MemberKey { + /// The key of a number: `None` for NaN, which equals no number. + fn of_num(number: Num) -> Option { + Some(match number { + Num::U(number) => MemberKey::Integer(i128::from(number)), + Num::I(number) => MemberKey::Integer(i128::from(number)), + Num::F(number) if number.is_nan() => return None, + // Exact: an integral f64 below 2^127 in magnitude is an `i128` + Num::F(number) if number.fract() == 0. && number.abs() < i128::MAX as f64 => { + MemberKey::Integer(number as i128) + } + // -0.0 is integral, so a float keyed by its bits is never zero + Num::F(number) => MemberKey::Float(number.to_bits()), + }) + } + + /// The key of a value of the document, read at most `depth` arrays or objects deep: + /// `None` for one nested deeper, or holding a number that equals no number, which equals + /// no allowed value nested at most `depth` deep. + fn of_json(value: &JsonValue, depth: usize) -> Option { + Some(match value { + JsonValue::Null => MemberKey::Null, + JsonValue::Bool(value) => MemberKey::Bool(*value), + JsonValue::Number(number) => return MemberKey::of_num(Num::of_json(number)), + JsonValue::String(value) => MemberKey::Text(value.clone()), + JsonValue::Array(values) => MemberKey::Array( + values + .iter() + .map(|value| MemberKey::of_json(value, depth.checked_sub(1)?)) + .collect::>()?, + ), + JsonValue::Object(map) => MemberKey::Object( + map.iter() + .map(|(name, value)| { + Some(( + name.clone(), + MemberKey::of_json(value, depth.checked_sub(1)?)?, + )) + }) + .collect::>()?, + ), + }) + } + + /// How many arrays or objects deep the key nests: 0 for a key of neither. + fn depth(&self) -> usize { + match self { + MemberKey::Array(keys) => 1 + keys.iter().map(MemberKey::depth).max().unwrap_or(0), + MemberKey::Object(entries) => { + 1 + entries + .iter() + .map(|(_, key)| key.depth()) + .max() + .unwrap_or(0) + } + _ => 0, + } + } + + /// The key of an `enum`'s allowed value: `None` for one no value of the document + /// equals, such as bytes or a map with a key that is not text. + fn of_value(value: &Value) -> Option { + Some(match value { + Value::Null => MemberKey::Null, + Value::Bool(value) => MemberKey::Bool(*value), + Value::Text(value) => MemberKey::Text(value.clone()), + Value::Array(values) => MemberKey::Array( + values + .iter() + .map(MemberKey::of_value) + .collect::>()?, + ), + Value::Map(entries) => MemberKey::Object( + entries + .iter() + .map(|(name, value)| { + Some((name.as_text()?.to_string(), MemberKey::of_value(value)?)) + }) + .collect::>()?, + ), + value => return MemberKey::of_num(Num::of_value(value)?), + }) + } +} diff --git a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/tests.rs b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/tests.rs index 18b550b3480..7f855dee203 100644 --- a/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/tests.rs +++ b/packages/rs-dpp/src/data_contract/document_type/schema/validation_schema/validate/v0/tests.rs @@ -20,7 +20,7 @@ fn first_violation( document: &JsonValue, ) -> Option { match validator - .validate(document, None) + .validate(document, None, None) .expect("expected the check to run") { None => None, @@ -722,7 +722,7 @@ fn should_refuse_a_value_nested_past_the_value_depth_limit() { let document = json!({ "p": deep }); let error = validator - .validate(&document, Some(3)) + .validate(&document, Some(3), None) .expect("expected the check to run") .expect("expected a refusal past the limit"); let ConsensusError::BasicError(BasicError::ValueError(value_error)) = &error else { @@ -736,7 +736,7 @@ fn should_refuse_a_value_nested_past_the_value_depth_limit() { // Under no limit the same value is not refused for its depth; the walk reports its own // verdict instead (the innermost value is not an array). let unlimited = validator - .validate(&document, None) + .validate(&document, None, None) .expect("expected the check to run"); assert!( !matches!( @@ -765,7 +765,7 @@ fn should_not_refuse_a_value_whose_container_nesting_equals_the_limit() { let document = json!({ "p": [[["s"]]] }); assert!( validator - .validate(&document, Some(3)) + .validate(&document, Some(3), None) .expect("expected the check to run") .is_none(), "a value at the container limit with a scalar leaf must not be refused for depth" @@ -773,7 +773,7 @@ fn should_not_refuse_a_value_whose_container_nesting_equals_the_limit() { // A fourth container exceeds the limit and is refused. let too_deep = json!({ "p": [[[["s"]]]] }); let error = validator - .validate(&too_deep, Some(3)) + .validate(&too_deep, Some(3), None) .expect("expected the check to run") .expect("expected a refusal past the limit"); assert!( @@ -889,3 +889,304 @@ fn should_compile_a_pattern_with_its_ecma_escapes_rewritten() { ); } } + +/// An `enum` of more than [`ENUM_SCAN_LIMIT`] allowed values, of every kind a value can +/// equal, which the walk looks values up in rather than compares them with in turn. +fn long_enum() -> JsonValue { + let mut members = vec![ + json!(null), + json!(true), + json!("a"), + json!(""), + json!(3), + json!(-4), + json!(1.5), + json!(-0.0), + json!(1e300), + json!(u64::MAX), + json!(i64::MIN), + json!([1, "a"]), + json!([]), + json!({ "a": 1, "b": [2.0] }), + json!({}), + ]; + members.extend((100..120).map(|member| json!(member))); + JsonValue::Array(members) +} + +#[test] +fn should_admit_a_value_of_a_long_enum_as_a_comparison_with_each_value_does() { + let members = long_enum(); + assert!(members.as_array().expect("a list").len() > ENUM_SCAN_LIMIT); + let validator = validator_of(&with_value(&json!({ "enum": members }))); + let allowed = Value::from(members); + let allowed = allowed.as_array().expect("a list"); + for value in [ + json!(null), + json!(true), + json!(false), + json!("a"), + json!("b"), + json!("3"), + json!(3), + json!(3.0), + json!(-4), + json!(-4.0), + json!(4), + json!(1.5), + json!(1.25), + json!(0), + json!(0.0), + json!(-0.0), + json!(1e300), + json!(1e299), + json!(u64::MAX), + json!(u64::MAX as f64), + json!(i64::MIN), + json!(i64::MIN as f64), + json!(9223372036854775807_i64), + json!([1, "a"]), + json!([1.0, "a"]), + json!(["a", 1]), + json!([1]), + json!([]), + json!({ "a": 1, "b": [2] }), + json!({ "a": 1 }), + json!({}), + json!(105), + json!(105.0), + json!(105.5), + json!(120), + ] { + let compared = allowed.iter().any(|member| equal_value(&value, member)); + let admitted = first_violation(&validator, &json!({ "value": value })).is_none(); + assert_eq!(admitted, compared, "{value}"); + } + // and as the existing validator decides, where it reads numbers alike + check(vec![ + (json!({ "enum": long_enum() }), json!(3.0), true), + (json!({ "enum": long_enum() }), json!(-0.0), true), + (json!({ "enum": long_enum() }), json!([1.0, "a"]), true), + ( + json!({ "enum": long_enum() }), + json!({ "a": 1, "b": [2] }), + true, + ), + (json!({ "enum": long_enum() }), json!("b"), false), + (json!({ "enum": long_enum() }), json!(120), false), + ]); +} + +#[test] +fn should_refuse_a_value_nested_deeper_than_every_allowed_value_of_a_long_enum() { + // The deepest allowed value of `long_enum` nests two levels deep + check(vec![ + ( + json!({ "enum": long_enum() }), + json!({ "a": 1, "b": [2.0] }), + true, + ), + ( + json!({ "enum": long_enum() }), + json!({ "a": 1, "b": [[2.0]] }), + false, + ), + (json!({ "enum": long_enum() }), json!([[[]]]), false), + (json!({ "enum": long_enum() }), json!([{ "a": {} }]), false), + ]); + // However deep the value nests, only as deep as the allowed values is read + let mut nested = json!(1); + for _ in 0..1000 { + nested = json!([nested]); + } + let validator = validator_of(&with_value(&json!({ "enum": long_enum() }))); + let error = first_violation(&validator, &json!({ "value": nested })) + .expect("expected the value to be refused"); + assert_eq!(error.keyword(), "enum"); +} + +/// The listing the latest protocol version reports an `enum` violation with. +fn latest_listing() -> Option { + let limits = &PlatformVersion::latest().system_limits; + limits + .max_enum_violation_listed_values + .zip(limits.max_enum_violation_listed_bytes) + .map(|(max_values, max_bytes)| EnumViolationListing { + max_values: usize::from(max_values), + max_bytes: usize::from(max_bytes), + }) +} + +/// The violation `validator` reports for `document` with `enum_listing`. +fn enum_violation( + validator: &DocumentSchemaValidator, + document: &JsonValue, + enum_listing: Option, +) -> JsonSchemaError { + match validator + .validate(document, None, enum_listing) + .expect("expected the check to run") + { + Some(ConsensusError::BasicError(BasicError::JsonSchemaError(error))) => error, + other => panic!("expected a JSON Schema error, got {other:?}"), + } +} + +#[test] +fn should_list_the_allowed_values_that_fit_and_count_the_rest() { + let members: Vec = (0..1000).map(|member| json!(member)).collect(); + let validator = validator_of(&with_value(&json!({ "enum": members }))); + let document = json!({ "value": -1 }); + let all = enum_violation(&validator, &document, None); + let bounded = enum_violation(&validator, &document, latest_listing()); + // The same violation, at the same place + assert_eq!(bounded.keyword(), "enum"); + assert_eq!(bounded.keyword(), all.keyword()); + assert_eq!(bounded.instance_path(), all.instance_path()); + assert_eq!(bounded.schema_path(), all.schema_path()); + let listed = |error: &JsonSchemaError| { + error + .params() + .get_optional_array_slice("enum") + .expect("a list") + .map(<[Value]>::to_vec) + }; + assert_eq!(listed(&all).map(|list| list.len()), Some(1000)); + // The first 16 allowed values, then a count of the others + let bounded_list = listed(&bounded).expect("expected a list"); + assert_eq!(bounded_list, listed(&all).expect("a list")[..16]); + assert_eq!( + bounded.error_summary(), + "-1 is not one of [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] or 984 more" + ); +} + +#[test] +fn should_report_a_short_enum_violation_as_without_a_bound() { + let validator = validator_of(&with_value(&json!({ "enum": ["a", "b", 3] }))); + let document = json!({ "value": "c" }); + let bounded = enum_violation(&validator, &document, latest_listing()); + assert_eq!(bounded, enum_violation(&validator, &document, None)); + assert_eq!( + bounded.error_summary(), + "\"c\" is not one of [\"a\",\"b\",3]" + ); +} + +#[test] +fn should_stop_listing_before_an_allowed_value_that_does_not_fit() { + let long = "x".repeat(300); + for (members, document, summary) in [ + // A value whose JSON takes more than the bytes left + ( + json!(["a", long, "b"]), + json!("c"), + "\"c\" is not one of [\"a\"] or 2 more", + ), + ( + json!([long, "b"]), + json!("c"), + "\"c\" is not one of [] or 2 more", + ), + // An array or an object, whatever its size + ( + json!([1, [2], 3]), + json!(4), + "4 is not one of [1] or 2 more", + ), + ( + json!([{ "a": 1 }, 3]), + json!(4), + "4 is not one of [] or 2 more", + ), + ] { + let validator = validator_of(&with_value(&json!({ "enum": members }))); + let error = enum_violation(&validator, &json!({ "value": document }), latest_listing()); + assert_eq!(error.keyword(), "enum"); + assert_eq!(error.error_summary(), summary, "{members}"); + } +} + +#[test] +fn should_list_the_allowed_values_whose_json_fits_the_bytes_together() { + let listing = latest_listing().expect("expected the latest version to bound the listing"); + // Strings whose JSON takes 32 bytes each, quotes included + let members: Vec = (0..20) + .map(|member| json!(format!("{member:030}"))) + .collect(); + let validator = validator_of(&with_value(&json!({ "enum": members }))); + let error = enum_violation(&validator, &json!({ "value": "c" }), Some(listing)); + let listed = error + .params() + .get_optional_array_slice("enum") + .expect("expected params") + .expect("expected a list") + .len(); + assert_eq!(listed, (listing.max_bytes / 32).min(listing.max_values)); + assert!(error + .error_summary() + .ends_with(&format!(" or {} more", 20 - listed))); +} + +#[test] +fn should_report_the_same_first_violation_whatever_the_enum_length() { + // Each document breaks the schema first where the walk reaches it first; a long enum + // is looked up rather than scanned, and that changes neither the verdict nor which + // violation comes first + let long: Vec = (0..100).map(|member| json!(format!("v{member}"))).collect(); + let root = json!({ + "type": "object", + "properties": { + "kind": { "type": "string", "enum": long }, + "tags": { + "type": "array", + "items": { "type": "string", "enum": long }, + "maxItems": 2, + "uniqueItems": true, + }, + "count": { "type": "integer", "maximum": 5 }, + }, + "required": ["kind"], + "additionalProperties": false, + }); + let validator = validator_of(&root); + for (document, keyword, instance_path) in [ + (json!({ "kind": "x" }), "enum", "/kind"), + (json!({ "kind": 1 }), "type", "/kind"), + ( + json!({ "kind": "v1", "tags": ["v1", "x"] }), + "enum", + "/tags/1", + ), + // An element outside the enum comes before too many elements + ( + json!({ "kind": "v1", "tags": ["x", "v1", "v2"] }), + "enum", + "/tags/0", + ), + ( + json!({ "kind": "v1", "tags": ["v1", "v2", "v3"] }), + "maxItems", + "/tags", + ), + ( + json!({ "kind": "v1", "tags": ["v1", "v1"] }), + "uniqueItems", + "/tags", + ), + (json!({ "count": 6 }), "maximum", "/count"), + ( + json!({ "kind": "v1", "other": 1 }), + "additionalProperties", + "", + ), + ] { + let violation = first_violation(&validator, &document).expect("expected a violation"); + assert_eq!( + (violation.keyword(), violation.instance_path()), + (keyword, instance_path), + "{document}" + ); + assert!(!existing_validator_accepts(&root, &document), "{document}"); + } +} diff --git a/packages/rs-dpp/src/data_contract/methods/validate_document/v2/mod.rs b/packages/rs-dpp/src/data_contract/methods/validate_document/v2/mod.rs index 8c728154087..27d9882cfd5 100644 --- a/packages/rs-dpp/src/data_contract/methods/validate_document/v2/mod.rs +++ b/packages/rs-dpp/src/data_contract/methods/validate_document/v2/mod.rs @@ -5,7 +5,7 @@ use crate::data_contract::accessors::v0::DataContractV0Getters; use crate::data_contract::document_type::accessors::DocumentTypeV0Getters; use crate::data_contract::document_type::property_constraints::DocumentSystemValues; use crate::data_contract::document_type::schema::validation_schema::{ - native_validator_of, DocumentSchemaValidator, + native_validator_of, DocumentSchemaValidator, EnumViolationListing, }; use crate::data_contract::document_type::DocumentTypeRef; use crate::data_contract::errors::DataContractError; @@ -59,6 +59,14 @@ impl DataContract { .system_limits .max_document_value_depth .map(|depth| depth as usize); + let limits = &platform_version.system_limits; + let enum_listing = limits + .max_enum_violation_listed_values + .zip(limits.max_enum_violation_listed_bytes) + .map(|(max_values, max_bytes)| EnumViolationListing { + max_values: usize::from(max_values), + max_bytes: usize::from(max_bytes), + }); Self::validate_document_properties_v1_checks( document_type, @@ -66,10 +74,12 @@ impl DataContract { system, platform_version, |json_value| { - Ok(match validator.validate(json_value, max_value_depth)? { - Some(error) => SimpleConsensusValidationResult::new_with_error(error), - None => SimpleConsensusValidationResult::new(), - }) + Ok( + match validator.validate(json_value, max_value_depth, enum_listing)? { + Some(error) => SimpleConsensusValidationResult::new_with_error(error), + None => SimpleConsensusValidationResult::new(), + }, + ) }, ) } diff --git a/packages/rs-dpp/src/data_contract/methods/validate_document/v2/tests.rs b/packages/rs-dpp/src/data_contract/methods/validate_document/v2/tests.rs index 18fbe1daee1..4edf911420f 100644 --- a/packages/rs-dpp/src/data_contract/methods/validate_document/v2/tests.rs +++ b/packages/rs-dpp/src/data_contract/methods/validate_document/v2/tests.rs @@ -1,5 +1,6 @@ //! Generation 2 of document validation, at protocol version 15 against generation 1 at 14. +use crate::consensus::basic::json_schema_error::JsonSchemaError; use crate::consensus::basic::BasicError; use crate::consensus::codes::ErrorWithCode; use crate::consensus::ConsensusError; @@ -951,3 +952,128 @@ fn should_refuse_every_document_of_a_type_holding_a_pattern_that_does_not_parse( "{errors:?}" ); } + +/// The document type `item` with a typed array `list` whose elements take one of `members` +/// integers, and a document whose `list` holds `elements` integers none of them takes. +fn enum_list_contract_and_document(members: u64, elements: u64) -> (DataContract, Value) { + let allowed: Vec = (0..members).map(Value::U64).collect(); + let contract = item_contract(platform_value!({ + "list": { + "type": "array", + "items": { "type": "integer", "enum": allowed }, + "maxItems": elements, + "position": 0, + }, + })); + let list: Vec = (0..elements) + .map(|element| Value::U64(members + element)) + .collect(); + (contract, platform_value!({ "list": list })) +} + +/// The JSON Schema error `error` is. +fn json_schema_error(error: &ConsensusError) -> &JsonSchemaError { + match error { + ConsensusError::BasicError(BasicError::JsonSchemaError(error)) => error, + other => panic!("expected a JSON Schema error, got {other:?}"), + } +} + +#[test] +fn should_report_one_small_enum_violation_for_a_document_breaking_a_long_enum_many_times() { + use crate::serialization::{ + PlatformDeserializableUntrusted, PlatformSerializableWithPlatformVersion, + }; + let latest = PlatformVersion::latest(); + let (contract, document) = enum_list_contract_and_document(300, 300); + let errors = document_errors(&contract, document, latest); + // The first violation only, the one consensus reports + assert_eq!(errors.len(), 1); + let error = json_schema_error(&errors[0]); + assert_eq!(error.keyword(), "enum"); + assert_eq!(error.instance_path(), "/list/0"); + let listed = error + .params() + .get_optional_array_slice("enum") + .expect("expected params") + .expect("expected a list") + .len(); + assert_eq!( + Some(listed), + latest + .system_limits + .max_enum_violation_listed_values + .map(usize::from) + ); + // A client can read the error back + let bytes = errors[0] + .serialize_to_bytes_with_platform_version(latest) + .expect("expected the error to serialize"); + assert_eq!( + ConsensusError::deserialize_from_bytes_untrusted(&bytes).expect("expected it to read back"), + errors[0] + ); +} + +#[test] +fn should_report_an_enum_violation_a_client_can_read_whatever_the_allowed_values_hold() { + use crate::serialization::{ + PlatformDeserializableUntrusted, PlatformSerializableWithPlatformVersion, + }; + let latest = PlatformVersion::latest(); + let long = "x".repeat(1500); + // Strings whose JSON fills the listing's bytes, the most a listing holds + let filling: Vec = (0..16) + .map(|member| Value::Text(format!("{member:014}"))) + .collect(); + let object = |last: u64| platform_value!({ "a": long.clone(), "b": [1, 2, last] }); + for allowed in [ + filling, + vec![Value::Text(long.clone()), Value::Text("b".to_string())], + vec![Value::Text("a".to_string()), Value::Text(long.clone())], + vec![object(3), object(4)], + ] { + let contract = item_contract(platform_value!({ + "label": { "type": "string", "enum": allowed, "position": 0 }, + })); + let errors = document_errors( + &contract, + platform_value!({ "label": "not an allowed value" }), + latest, + ); + assert_eq!(errors.len(), 1); + assert_eq!(json_schema_error(&errors[0]).keyword(), "enum"); + let bytes = errors[0] + .serialize_to_bytes_with_platform_version(latest) + .expect("expected the error to serialize"); + assert_eq!( + ConsensusError::deserialize_from_bytes_untrusted(&bytes) + .expect("expected it to read back"), + errors[0] + ); + } +} + +#[test] +fn should_report_an_enum_violation_at_protocol_version_14_as_before() { + let protocol_version_14 = PlatformVersion::get(14).expect("expected protocol version 14"); + let (contract, document) = enum_list_contract_and_document(300, 2); + let errors = document_errors(&contract, document.clone(), protocol_version_14); + let first = json_schema_error(errors.first().expect("expected an error")); + assert_eq!(first.keyword(), "enum"); + // Every allowed value listed, as the JSON Schema validator lists them + assert_eq!( + first + .params() + .get_optional_array_slice("enum") + .expect("expected params") + .map(<[Value]>::len), + Some(300) + ); + // with the code protocol version 15 reports + let latest = document_errors(&contract, document, PlatformVersion::latest()); + assert_eq!( + latest.first().map(ErrorWithCode::code), + errors.first().map(ErrorWithCode::code) + ); +} diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 19ff20f0c9f..98d5f56693f 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -599,6 +599,8 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { fee_version: FEE_VERSION1, system_limits: SystemLimits { max_document_revisions_erased_per_transition: None, + max_enum_violation_listed_values: None, + max_enum_violation_listed_bytes: None, credit_pool_nullifier_backfill_page_size: 0, credit_pool_nullifier_backfill_batch_size: 0, estimated_contract_max_serialized_size: 16384, diff --git a/packages/rs-platform-version/src/version/system_limits/mod.rs b/packages/rs-platform-version/src/version/system_limits/mod.rs index c572525e802..d7097580e6d 100644 --- a/packages/rs-platform-version/src/version/system_limits/mod.rs +++ b/packages/rs-platform-version/src/version/system_limits/mod.rs @@ -9,6 +9,17 @@ pub struct SystemLimits { /// Maximum retained revisions removed by one erase, read by /// `erase_document_for_contract_operations` 0; absent before lifecycle activation. pub max_document_revisions_erased_per_transition: Option, + /// Most allowed values a document's `enum` violation lists in its parameters and its + /// message, the first the schema writes; the others are counted, not listed. Read by the + /// native document validator (`validation_schema` 0, which `validate_document` 2 of + /// protocol version 15 runs). `None` in the earlier tables, whose document validation + /// never reaches it. + pub max_enum_violation_listed_values: Option, + /// Most bytes the JSON of the allowed values a document's `enum` violation lists takes + /// together; listing stops before the first value that would go past it, or that is an + /// array or an object. Read with `max_enum_violation_listed_values`, and `None` where it + /// is. + pub max_enum_violation_listed_bytes: Option, /// Maximum ordered note page read by the CREDIT nullifier activation migration. pub credit_pool_nullifier_backfill_page_size: u16, /// Maximum insert operations per batch in the CREDIT nullifier activation migration. diff --git a/packages/rs-platform-version/src/version/system_limits/v1.rs b/packages/rs-platform-version/src/version/system_limits/v1.rs index d4c2f8c9b3b..148502fbc40 100644 --- a/packages/rs-platform-version/src/version/system_limits/v1.rs +++ b/packages/rs-platform-version/src/version/system_limits/v1.rs @@ -2,6 +2,8 @@ use crate::version::system_limits::SystemLimits; pub const SYSTEM_LIMITS_V1: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: None, + max_enum_violation_listed_values: None, + max_enum_violation_listed_bytes: None, credit_pool_nullifier_backfill_page_size: 0, credit_pool_nullifier_backfill_batch_size: 0, estimated_contract_max_serialized_size: 16384, diff --git a/packages/rs-platform-version/src/version/system_limits/v2.rs b/packages/rs-platform-version/src/version/system_limits/v2.rs index 9a74de90d71..c0570d185ce 100644 --- a/packages/rs-platform-version/src/version/system_limits/v2.rs +++ b/packages/rs-platform-version/src/version/system_limits/v2.rs @@ -7,6 +7,8 @@ use crate::version::system_limits::SystemLimits; /// the bare asset-unlock transaction fee and too low a minimum for a Core `TxOut`. pub const SYSTEM_LIMITS_V2: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: None, + max_enum_violation_listed_values: None, + max_enum_violation_listed_bytes: None, credit_pool_nullifier_backfill_page_size: 0, credit_pool_nullifier_backfill_batch_size: 0, estimated_contract_max_serialized_size: 16384, diff --git a/packages/rs-platform-version/src/version/system_limits/v3.rs b/packages/rs-platform-version/src/version/system_limits/v3.rs index da29418d9b7..09a2ac99f20 100644 --- a/packages/rs-platform-version/src/version/system_limits/v3.rs +++ b/packages/rs-platform-version/src/version/system_limits/v3.rs @@ -8,6 +8,8 @@ use crate::version::system_limits::SystemLimits; /// live on networks without it. pub const SYSTEM_LIMITS_V3: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: None, + max_enum_violation_listed_values: None, + max_enum_violation_listed_bytes: None, credit_pool_nullifier_backfill_page_size: 0, credit_pool_nullifier_backfill_batch_size: 0, estimated_contract_max_serialized_size: 16384, diff --git a/packages/rs-platform-version/src/version/system_limits/v4.rs b/packages/rs-platform-version/src/version/system_limits/v4.rs index 6ef0d5f2efa..e80603250ad 100644 --- a/packages/rs-platform-version/src/version/system_limits/v4.rs +++ b/packages/rs-platform-version/src/version/system_limits/v4.rs @@ -127,6 +127,8 @@ use crate::version::system_limits::SystemLimits; /// (`max_rekey_batch_moves`). `None` in the earlier tables, which never run the rewrite. pub const SYSTEM_LIMITS_V4: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: None, + max_enum_violation_listed_values: None, + max_enum_violation_listed_bytes: None, // The page matches one notes-tree chunk (2^11). A batch holds at most that // page's missing rhos; GroveDB accepts a Vec of operations with no smaller API cap. credit_pool_nullifier_backfill_page_size: 2048, diff --git a/packages/rs-platform-version/src/version/system_limits/v5.rs b/packages/rs-platform-version/src/version/system_limits/v5.rs index ffe2e8d18e0..96b29f5f4b4 100644 --- a/packages/rs-platform-version/src/version/system_limits/v5.rs +++ b/packages/rs-platform-version/src/version/system_limits/v5.rs @@ -22,8 +22,14 @@ use crate::version::system_limits::SystemLimits; /// * Retained-history erasure: one transition removes at most 100 revisions /// (`max_document_revisions_erased_per_transition`, `None` before), as enforced by /// `erase_document_for_contract_operations` 0. +/// * Enum violations: a document's `enum` violation lists at most 16 of the allowed values +/// (`max_enum_violation_listed_values`, `None` before), whose JSON takes at most 256 bytes +/// together (`max_enum_violation_listed_bytes`, `None` before), and counts the others, so +/// the error stays small, and readable by a client, whatever the schema's enum holds. pub const SYSTEM_LIMITS_V5: SystemLimits = SystemLimits { max_document_revisions_erased_per_transition: Some(100), // one bounded lifecycle erasure chunk (note 41) + max_enum_violation_listed_values: Some(16), // changed: an enum violation lists at most 16 allowed values + max_enum_violation_listed_bytes: Some(256), // changed: whose JSON takes at most 256 bytes together max_compiled_contract_size: Some(1_048_576), // changed: contracts stored compiled take at most 1 MiB max_private_document_entry_size: Some(4096), // changed: a private document type's entries take at most 4096 bytes private_document_store_target_chunk_bytes: Some(1024 * 1024), // changed: a private type's store keeps a compacted chunk near 1 MiB diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index 4407188f234..174d7bb517e 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -146,7 +146,7 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// `validate_document` 2, which checks a document against its type's typed /// validation schema (`document_type::schema::validation_schema`, built by /// `validation_schema` 0) instead of the `jsonschema` crate, reaching the same verdicts and error codes but for -/// two rules of its own. First, in a `uniqueItems` list, `0.0` and `-0.0` are +/// two rules of its own, and reporting an `enum` violation in a bounded form. First, in a `uniqueItems` list, `0.0` and `-0.0` are /// one value however long the list. Second, a `format: regex` value is /// checked for its syntax alone: whether it parses as a regular expression, /// not whether it compiles, since no consensus path uses the value as a @@ -163,6 +163,11 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// Protocol v14 refused every value reaching the schema holding such a /// pattern; a value that is not a string, which no pattern applies to, is /// now checked against the rest of that schema alone. +/// An `enum` violation lists the leading allowed values that fit +/// `max_enum_violation_listed_values` and `max_enum_violation_listed_bytes` +/// (`SYSTEM_LIMITS_V5`), none an array or an object, and counts the others, +/// where protocol v14 lists every allowed value; its keyword, paths and code +/// are unchanged, and an `enum` of a few short values reads as before. /// A document type whose schema the validation /// schema does not represent has its documents refused /// (`InvalidContractStructure`): no document is checked through the @@ -954,7 +959,7 @@ pub const PLATFORM_V15: PlatformVersion = PlatformVersion { }, system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V4, // changed: DPNS v3 checks a name's create with keywords (note 3); the decision proposals contract (note 12); DashPay v3 states what its data triggers checked (note 24) fee_version: FEE_VERSION4, // changed: decision proposals prefund their poll with 0.5 Dash (note 12); a private document entry pays its storage and 10,000,000 credits (note 25); a multiple choice proposal pays a 0.2 Dash processing fee on create (note 40) - system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41) + system_limits: SYSTEM_LIMITS_V5, // changed: max_compiled_contract_size caps a contract stored compiled at 1 MiB (note 8); max_private_document_entry_size caps a private type's entries at 4096 bytes, and the private document store limits size its store (note 13); max_private_document_entries_per_transition caps a private document create at 16 entries (note 25); max_document_revisions_erased_per_transition caps an erase at 100 revisions (note 41); max_enum_violation_listed_values and max_enum_violation_listed_bytes list at most 16 of a document enum violation's allowed values, in at most 256 bytes (note 6) consensus: ConsensusVersions { tenderdash_consensus_version: 1, evidence: Some(ConsensusEvidenceParams {