diff --git a/jamcodec/encoding.py b/jamcodec/encoding.py new file mode 100644 index 0000000..46cd03b --- /dev/null +++ b/jamcodec/encoding.py @@ -0,0 +1,383 @@ +"""Prepare standard model values for binary encoding without codec wrappers.""" + +import dataclasses +import struct +from types import GetSetDescriptorType + +from jamcodec import types +from jamcodec.base import JamBytes, JamCodecType + + +_WRAPPERS = { + types.UnsignedInteger: JamCodecType, + types.SignedInteger: JamCodecType, + types.Float: JamCodecType, + types.Bool: JamCodecType, + types.NullType: JamCodecType, + types.CompactDef: JamCodecType, + types.BitVec: JamCodecType, + types.BitArray: JamCodecType, + types.BytesDef: JamCodecType, + types.StringDef: JamCodecType, + types.VarInt64Def: JamCodecType, + types.HashDef: types.HashDefObject, + types.Struct: types.StructObject, + types.Tuple: JamCodecType, + types.Enum: JamCodecType, + types.Option: JamCodecType, + types.Vec: types.VecType, + types.BoundedVec: types.VecType, + types.Array: types.ArrayObject, + types.Map: JamCodecType, +} +_TYPE_METHODS = { + cls: (cls.new, cls.deserialize, cls.serialize, cls.encode, + getattr(cls, '_check_count', None)) + for cls in _WRAPPERS +} +_WRAPPER_METHODS = { + cls: (cls.__init__, cls.deserialize, cls.serialize, cls.encode, + cls.__getattribute__, cls.__setattr__, cls.value) + for cls in set(_WRAPPERS.values()) +} +_HOOK_NAMES = frozenset(('new', 'deserialize', 'serialize', 'encode', '_check_count')) +_PRIMITIVES = frozenset(_WRAPPERS) - { + types.Struct, types.Tuple, types.Enum, types.Option, + types.Vec, types.BoundedVec, types.Array, types.Map, +} +_INTEGERS = frozenset((types.UnsignedInteger, types.SignedInteger, types.CompactDef, types.VarInt64Def)) +_MODEL_HOOKS = frozenset(('__getattr__', '__bool__', '__len__')) + + +class _UseWrappers(Exception): + pass + + +class _Encoder: + def __init__(self): + self.checked = set() + self.checked_types = set() + self.checked_wrappers = set() + self.model_fields = {} + self.active = set() + self.data = bytearray() + + def check(self, codec): + cls = type(codec) + if id(codec) in self.checked: + return cls + wrapper = _WRAPPERS.get(cls) + if (wrapper is None or codec.scale_type_cls is not wrapper + or not _HOOK_NAMES.isdisjoint(codec.__dict__)): + raise _UseWrappers + if cls not in self.checked_types: + if (cls.new, cls.deserialize, cls.serialize, cls.encode, + getattr(cls, '_check_count', None)) != _TYPE_METHODS[cls]: + raise _UseWrappers + self.checked_types.add(cls) + if wrapper not in self.checked_wrappers: + if (wrapper.__init__, wrapper.deserialize, wrapper.serialize, wrapper.encode, + wrapper.__getattribute__, wrapper.__setattr__, wrapper.value) != _WRAPPER_METHODS[wrapper]: + raise _UseWrappers + self.checked_wrappers.add(wrapper) + + if cls is types.Struct: + if type(codec.arguments) is not dict or any(type(name) is not str for name in codec.arguments): + raise _UseWrappers + elif cls is types.Enum: + if (type(codec.variants) is not dict or + (codec._deserialize_type is not None and type(codec._deserialize_type) is not type)): + raise _UseWrappers + if any(type(name) is not str for name in codec.variants): + raise _UseWrappers + elif cls is types.Tuple: + if type(codec.values) not in (list, tuple): + raise _UseWrappers + elif cls in (types.Vec, types.BoundedVec, types.Array): + if cls is types.Array and type(codec.length) is not int: + raise _UseWrappers + if cls is types.BoundedVec and any(type(bound) is not int for bound in ( + codec.minimum, codec.maximum, codec.multiple, + )): + raise _UseWrappers + if cls in (types.Vec, types.BoundedVec): + self.check(types.VarInt64) + if codec.type_def is types.U8: + self.check(types.Bytes) + elif cls is types.Map: + self.check(types.VarInt64) + elif cls in (types.BytesDef, types.StringDef, types.BitVec): + self.check(types.VarInt64) + elif cls in (types.UnsignedInteger, types.SignedInteger, types.HashDef): + if (type(codec.byte_count) is not int + or type(codec.bits) is not int): + raise _UseWrappers + elif cls is types.Float: + if type(codec.struct_format) is not str: + raise _UseWrappers + elif cls is types.BitArray: + if type(codec.length) is not int or type(codec.strict_decoding) is not bool: + raise _UseWrappers + self.checked.add(id(codec)) + return cls + + def fields(self, value): + """Read plain dataclass fields without invoking user-defined accessors.""" + cls = type(value) + if type(cls) is not type or not dataclasses.is_dataclass(cls): + raise _UseWrappers + if cls not in self.model_fields: + if (cls.__getattribute__ is not object.__getattribute__ or + issubclass(cls, JamCodecType)): + raise _UseWrappers + names = tuple(field.name for field in dataclasses.fields(cls)) + remaining = set(names) + dictionary = None + for base in cls.__mro__: + attributes = vars(base) + if not _MODEL_HOOKS.isdisjoint(attributes): + raise _UseWrappers + if dictionary is None and '__dict__' in attributes: + dictionary = attributes['__dict__'] + for name in remaining.intersection(attributes): + if hasattr(type(attributes[name]), '__get__'): + raise _UseWrappers + remaining.remove(name) + if type(dictionary) is not GetSetDescriptorType: + raise _UseWrappers + self.model_fields[cls] = names + values = object.__getattribute__(value, '__dict__') + names = self.model_fields[cls] + if any(name not in values for name in names): + raise _UseWrappers + return values if len(values) == len(names) else {name: values[name] for name in names} + + def plain_leaf(self, value): + cls = type(value) + if cls in (bytes, bytearray, int, float, bool, str, type(None)): + return + if value is types.Null and not any( + name in base.__dict__ for base in cls.__mro__ for name in ('__bool__', '__len__') + ): + return + if id(value) in self.active: + raise _UseWrappers + self.active.add(id(value)) + try: + if cls in (list, tuple): + for item in value: + self.plain_leaf(item) + elif cls is dict: + for key, item in value.items(): + self.plain_leaf(key) + self.plain_leaf(item) + else: + raise _UseWrappers + finally: + self.active.remove(id(value)) + + def primitive(self, codec, cls, value, native_bytes): + value_type = type(value) + if cls in _INTEGERS: + if value_type is not int: + raise _UseWrappers + return value + if cls is types.Float: + if value_type is not float: + raise _UseWrappers + return value + if cls is types.Bool: + if value_type is not bool: + raise _UseWrappers + return value + if cls in (types.BytesDef, types.HashDef): + if value_type is bytearray: + value = bytes(value) + value_type = bytes + if value_type is not bytes: + raise _UseWrappers + if cls is types.HashDef and len(value) != codec.byte_count: + raise _UseWrappers + return value if native_bytes else codec.serialize(value) + if cls is types.StringDef: + if value_type is not str: + raise _UseWrappers + return value + # Less common leaves retain their existing normalization and validation. + self.plain_leaf(value) + return codec.serialize(codec.deserialize(value)) + + def length(self, count): + if count < 128: + self.data.append(count) + else: + self.data.extend(types.VarInt64.encode(count).data) + + def prepare(self, codec, value, emit=True, native_bytes=True): + """Write plain inputs, or return serialized values for map sorting. + + When emitting a container, only its None/non-None result matters to + Option; its bytes have already been appended to the output buffer. + """ + cls = self.check(codec) + if cls in _PRIMITIVES: + result = self.primitive(codec, cls, value, native_bytes) + if emit: + if cls is types.HashDef: + self.data.extend(result) + elif cls is types.UnsignedInteger: + self.data.extend(result.to_bytes(codec.byte_count, 'little')) + elif cls is types.BytesDef: + self.length(len(result)) + self.data.extend(result) + elif cls is types.Bool: + self.data.append(result) + elif cls is types.VarInt64Def and 0 <= result < 128: + self.data.append(result) + elif cls is not types.NullType: + self.data.extend(codec.encode(result).data) + return result + if cls is types.Struct: + if type(value) is not dict: + value = self.fields(value) + elif any(type(name) is not str for name in value): + raise _UseWrappers + result = {} if not emit else True + for name, child in codec.arguments.items(): + if name not in value: + raise _UseWrappers + item = self.prepare(child, value[name], emit, native_bytes) + if not emit: + result[name] = item + return result + if cls is types.Tuple: + if type(value) not in (tuple, list) or len(value) < len(codec.values): + raise _UseWrappers + result = [self.prepare(child, value[index], emit, native_bytes) + for index, child in enumerate(codec.values)] + return True if emit else tuple(result) + if cls is types.Option: + start = len(self.data) + if emit: + self.data.append(0) + result = None if value is None else self.prepare(codec.some, value, emit, native_bytes) + if emit and result is not None: + self.data[start] = 1 + elif emit: + # Option(Null) and nested Options normalize to None too. + del self.data[start + 1:] + return result + if cls is types.Enum: + if type(value) not in (dict, str): + fields = self.fields(value) + if codec._deserialize_type and isinstance(value, codec._deserialize_type): + value = {name: item for name, item in fields.items() if item is not None} + else: + raise _UseWrappers + if type(value) is str: + value = {value: None} + if len(value) != 1: + raise _UseWrappers + name, item = next(iter(value.items())) + if type(name) is not str: + raise _UseWrappers + for index, (variant, child) in enumerate(codec.variants.items()): + if type(child) is dict: + index, child = child.get('id', index), child.get('type', child) + if variant == name: + if type(index) is not int or not 0 <= index <= 255: + raise _UseWrappers + if emit: + self.data.append(index) + if child is None: + return name + item = self.prepare(child, item, emit, native_bytes) + return True if emit else {name: item} + raise _UseWrappers + if cls in (types.Vec, types.BoundedVec, types.Array): + array = cls is types.Array + if type(value) is bytearray: + value = bytes(value) + if type(value) not in (list, tuple, bytes): + raise _UseWrappers + if array and (type(value) is tuple or len(value) != codec.length): + raise _UseWrappers + if type(value) is list and not value: + if cls is types.BoundedVec: + codec._check_count(0) + if emit and not array: + self.data.append(0) + return True if emit else [] + if not array and codec.type_def is types.U8 and type(value) is not bytes: + raise _UseWrappers + if cls is types.BoundedVec: + codec._check_count(len(value)) + if codec.type_def is types.U8 and type(value) is bytes: + result = value if native_bytes else codec.serialize(value) + if emit: + if not array: + self.length(len(value)) + self.data.extend(value) + return result + if type(value) not in (list, tuple) or (array and type(value) is not list): + raise _UseWrappers + child_class = self.check(codec.type_def) + if (emit and child_class is types.HashDef + and all(type(item) is bytes and len(item) == codec.type_def.byte_count for item in value)): + if not array: + self.length(len(value)) + self.data.extend(b''.join(value)) + return True + if emit and child_class in _INTEGERS and all(type(item) is int for item in value): + if child_class is types.UnsignedInteger: + if not array: + self.length(len(value)) + for item in value: + self.data.extend(item.to_bytes(codec.type_def.byte_count, 'little')) + else: + self.data.extend(codec.encode(list(value) if type(value) is tuple else value).data) + return True + if emit and not array: + self.length(len(value)) + result = [self.prepare(codec.type_def, item, emit, native_bytes) for item in value] + return True if emit else result + if type(value) not in (dict, list, tuple): + raise _UseWrappers + if not value: + if emit: + self.data.append(0) + return [] + # Maps sort serialized representations, including values for duplicate + # keys. Keep that representation and let the existing encoder sort it. + items = value.items() if type(value) is dict else value + result = [] + for pair in items: + if type(pair) not in (tuple, list) or len(pair) != 2: + raise _UseWrappers + key, item = pair + result.append((self.prepare(codec.key_def, key, False, False), + self.prepare(codec.value_def, item, False, False))) + if emit: + self.data.extend(codec.encode(result).data) + return result + + +def encode_from_object(codec, value): + """Return binary data for standard inputs, or None to use normal wrappers.""" + # Bypassing wrappers pays off for collection models. Small mixed records + # are cheaper through the existing normalization path. + if (type(codec) is not types.Struct or type(codec.arguments) is not dict + or not codec.arguments or not all(type(child) in ( + types.Array, types.Vec, types.BoundedVec, types.Map, + ) for child in codec.arguments.values())): + return None + encoder = _Encoder() + try: + encoder.prepare(codec, value) + except (_UseWrappers, ValueError, TypeError, ArithmeticError, IndexError, + KeyError, AttributeError, RecursionError, struct.error): + # Probing uses only built-in hooks and plain inputs. On invalid or + # unsupported data, repeat the original path so normalization errors + # still take precedence over encode-time errors in earlier fields. + return None + return JamBytes(encoder.data) diff --git a/jamcodec/mixins.py b/jamcodec/mixins.py index 07712bf..596ba21 100644 --- a/jamcodec/mixins.py +++ b/jamcodec/mixins.py @@ -8,6 +8,7 @@ from jamcodec.base import JamCodecTypeDef, JamCodecType, JamBytes from jamcodec.types import Struct, StructObject, Option, Vec, Enum from jamcodec.decoding import decode_to_object +from jamcodec.encoding import encode_from_object T = TypeVar('T') @@ -69,7 +70,17 @@ def from_codec_type(cls: Type[T], scale_type: JamCodecType) -> T: return scale_type.to_serializable_obj() def to_jam_bytes(self) -> JamBytes: - scale_obj = self.to_codec_type() + to_codec_type = self.to_codec_type + if (getattr(to_codec_type, '__func__', None) is _DEFAULT_TO_CODEC_TYPE + and is_dataclass(self)): + codec = self.to_codec_def() + data = encode_from_object(codec, self) + if data is not None: + return data + scale_obj = codec.new() + scale_obj.deserialize(self) + else: + scale_obj = to_codec_type() if type(scale_obj) in (JamCodecType, StructObject): # The value is already normalized; only the encoded bytes are needed. return scale_obj.type_def.encode(scale_obj.value_serialized) @@ -106,3 +117,6 @@ def enum_value(self): for field in dataclasses.fields(self): if getattr(self, field.name, None) is not None: return field.name, getattr(self, field.name) + + +_DEFAULT_TO_CODEC_TYPE = Serializable.to_codec_type diff --git a/test/test_direct_encode.py b/test/test_direct_encode.py new file mode 100644 index 0000000..3e3db4e --- /dev/null +++ b/test/test_direct_encode.py @@ -0,0 +1,440 @@ +from dataclasses import dataclass, field +from types import MethodType +from unittest.mock import patch + +import pytest + +from jamcodec.base import JamBytes, JamCodecType +from jamcodec.encoding import encode_from_object +from jamcodec.exceptions import ScaleDeserializeException +from jamcodec.mixins import Serializable +from jamcodec.types import ( + Array, BitArray, Bool, BoundedVec, Bytes, BytesDef, Enum, F32, F64, H256, I16, Map, Null, + Option, Struct, StructObject, Tuple, U8, U16, U32, UnsignedInteger, VarInt64, Vec, +) + + +def legacy_encode(value): + """The binary encoding path before direct normalization was introduced.""" + wrapper = value.to_codec_type() + if type(wrapper) in (JamCodecType, StructObject): + return wrapper.type_def.encode(wrapper.value_serialized) + return wrapper.encode() + + +def assert_matches_legacy(value): + assert_encoding_matches_legacy(value) + # Exercise the collection path too when the original model is a mixed + # record, which deliberately keeps the normal wrapper path at the root. + assert_encoding_matches_legacy(model(Vec(value.to_codec_def()), [value])) + + +def assert_encoding_matches_legacy(value): + try: + expected = legacy_encode(value) + except Exception as error: + with pytest.raises(type(error)) as caught: + value.to_jam_bytes() + assert str(caught.value) == str(error) + else: + actual = value.to_jam_bytes() + assert actual.to_bytes() == expected.to_bytes() + assert actual.offset == 0 + assert actual.length == expected.length + + +@dataclass +class Item(Serializable): + number: int = field(metadata={'codec': U16}) + payload: bytes = field(metadata={'codec': Bytes}) + + +@dataclass +class Choice(Serializable): + item: object = None + empty: object = None + _codec_type_def = Enum(item={'id': 17, 'type': Item.to_codec_def()}, empty={'id': 255, 'type': Null}) + + +@dataclass +class Packet(Serializable): + records: list = field(metadata={'codec': BoundedVec(Item.to_codec_def(), 3, minimum=1)}) + optional: object = field(metadata={'codec': Option(Item.to_codec_def())}) + lookup: dict = field(metadata={'codec': Map(Tuple(H256, U16), Option(Bytes))}) + pair: tuple = field(metadata={'codec': Tuple(U8, Array(U16, 2))}) + fixed: bytes = field(metadata={'codec': Array(U8, 2)}) + bits: list = field(metadata={'codec': BitArray(9)}) + choice: Choice = field(metadata={'codec': Choice.to_codec_def()}) + + +@pytest.mark.parametrize('choice', [Choice(item=Item(7, b'item')), Choice(empty=Null)]) +def test_standard_nested_models_use_direct_normalization(choice): + value = Packet( + [Item(1, b'first'), Item(258, b'')], Item(65535, b'optional'), + {(bytes(32), 2): None, (b'\xff' * 32, 1): b'value'}, + (3, [1, 258]), bytearray(b'ab'), [True] * 9, choice, + ) + collection = model(Vec(Packet.to_codec_def()), [value]) + assert encode_from_object(collection.to_codec_def(), collection) is not None + assert_matches_legacy(value) + + +def model(codec, value): + @dataclass + class Value(Serializable): + item: object = field(metadata={'codec': codec}) + + return Value(value) + + +@pytest.mark.parametrize('codec,value', [ + (Array(U8, 2), [1, 255]), + (Array(U8, 2), 'é'), + (Array(U8, 2), '0x01ff'), + (Array(U16, 2), [1, 258]), + (Tuple(U16), 258), + (Tuple(U8, U16), [1, 258, 999]), + (Vec(U16), (1, 258)), + (Vec(U8), 'é'), + (BoundedVec(U8, 2, minimum=2, multiple=2), 'é'), + (Option(Null), Null), + (Option(Option(U16)), None), + (Bytes, [0, 255]), + (Bytes, '0x00ff'), + (H256, bytearray(range(32))), + (BitArray(9), [False] * 9), +]) +def test_accepted_input_representations_match(codec, value): + assert_matches_legacy(model(codec, value)) + + +@pytest.mark.parametrize('codec,value', [ + (U8, True), (U8, '1'), (U8, 256), (H256, b'short'), (H256, 7), + (Array(U8, 2), [256, 1]), (Array(U8, 2), [True, 1]), + (Array(U8, 2), b'x'), (Array(U16, 2), b'ab'), (Tuple(U8, U16), [1]), + (BoundedVec(U16, 4, minimum=2, multiple=2), [1, 2, 3]), + (BoundedVec(U8, 2, minimum=2), 'éa'), (Bytes, '0xgg'), + (BitArray(9), ['x']), +]) +def test_validation_errors_match(codec, value): + assert_matches_legacy(model(codec, value)) + + +@pytest.mark.parametrize('first,second,first_codec,second_codec', [ + (256, 7, U8, H256), + (256, '0xgg', U8, Bytes), + (b'short', 'not an integer', H256, U8), + ([256, 1], 'not an integer', Array(U8, 2), U8), +]) +def test_all_normalization_finishes_before_binary_validation(first, second, first_codec, second_codec): + @dataclass + class Invalid(Serializable): + first: object = field(metadata={'codec': first_codec}) + second: object = field(metadata={'codec': second_codec}) + + assert_matches_legacy(Invalid(first, second)) + + +@pytest.mark.parametrize('first,first_codec', [ + ([256, 1], Vec(U8)), + ([256, 1], BoundedVec(U8, 1)), + (1e100, F32), + (-1, VarInt64), + ([(1, None), (1, b'x')], Map(U8, Option(Bytes))), +]) +def test_early_binary_errors_defer_to_later_normalization_errors(first, first_codec): + @dataclass + class Invalid(Serializable): + first: object = field(metadata={'codec': first_codec}) + second: object = field(metadata={'codec': H256}) + + assert_matches_legacy(Invalid(first, 7)) + + +@pytest.mark.parametrize('codec,value', [ + (Vec(U8), b'abc'), + (Array(U8, 2), [1, 255]), + (Array(U16, 2), [1, 65535]), + (Array(H256, 2), [bytes(32), b'\xff' * 32]), + (BoundedVec(H256, 3), [bytes(32)]), + (Vec(I16), [-32768, 32767]), + (Vec(F32), [0.5, float('inf')]), + (Vec(F64), [0.5, float('nan')]), + (Tuple(Bool(), U32), (True, 2**32 - 1)), + (Option(Option(Null)), Null), +]) +def test_native_values_use_the_direct_path(codec, value): + value = model(codec, value) + value = model(Vec(value.to_codec_def()), [value]) + assert encode_from_object(value.to_codec_def(), value) == legacy_encode(value) + + +@pytest.mark.parametrize('codec,value', [ + (Map(Bytes, Bytes), [(b'key', b'z'), (b'key', b'a')]), + (Map(Option(Bytes), U8), [(None, 1), (b'x', 2)]), + (Map(U8, Option(H256)), [(1, None), (1, bytes(32))]), + (Map(Array(U8, 2), U8), [(bytearray(b'ab'), 1), (b'aa', 2)]), + (Map(U8, Enum(a=None, b=None)), [(1, 'b'), (1, 'a')]), + (Map(U8, U8), [(1, 2, 3)]), +]) +def test_map_sorting_uses_the_legacy_serialized_representations(codec, value): + assert_matches_legacy(model(codec, value)) + + +def test_custom_codec_hooks_run_once_and_in_the_original_order(): + events = [] + + class CustomByte(UnsignedInteger): + def deserialize(self, value): + events.append('deserialize') + return super().deserialize(value) + 1 + + def serialize(self, value): + events.append('serialize') + return value + 10 + + def encode(self, value): + events.append('encode') + return super().encode(value) + + @dataclass + class Value(Serializable): + first: bytes = field(metadata={'codec': Bytes}) + custom: int = field(metadata={'codec': CustomByte(8)}) + + value = Value(b'first', 2) + expected = legacy_encode(value) + events.clear() + assert value.to_jam_bytes() == expected + assert events == ['deserialize', 'serialize', 'encode'] + + +def test_custom_root_wrapper_keeps_post_encode_validation(): + events = [] + + class Wrapper(StructObject): + def deserialize(self, value): + events.append(('deserialize', self._data is not None)) + return super().deserialize(value) + + def encode(self, value=None): + events.append(('encode',)) + return JamBytes(b'\xaa' + super().encode(value).to_bytes()) + + @dataclass + class Value(Serializable): + number: int + _codec_type_def = Struct(number=U8).impl(Wrapper) + + expected = legacy_encode(Value(7)) + events.clear() + assert Value(7).to_jam_bytes() == expected + assert events == [('deserialize', False), ('encode',), ('deserialize', True)] + + +def test_custom_to_codec_type_is_called_once(): + calls = [] + + @dataclass + class Value(Serializable): + number: int = field(metadata={'codec': U8}) + + def to_codec_type(self): + calls.append('to_codec_type') + return super().to_codec_type() + + assert Value(7).to_jam_bytes().to_bytes() == b'\x07' + assert calls == ['to_codec_type'] + + +def test_replaced_base_to_codec_type_is_called_once(): + original = Serializable.to_codec_type + calls = [] + + def custom(self): + calls.append('to_codec_type') + return original(self) + + with patch.object(Serializable, 'to_codec_type', custom): + assert model(Vec(U16), [1, 258]).to_jam_bytes() == JamBytes(b'\x02\x01\x00\x02\x01') + assert calls == ['to_codec_type'] + + +def test_mutable_definitions_and_wrapper_changes_are_not_cached_between_calls(): + @dataclass + class Value(Serializable): + number: int + _codec_type_def = Struct(number=U8) + + assert Value(7).to_jam_bytes().to_bytes() == b'\x07' + Value.to_codec_def().arguments['number'] = U16 + assert Value(258).to_jam_bytes().to_bytes() == b'\x02\x01' + + class Prefix(StructObject): + def encode(self, value=None): + return JamBytes(b'\xaa' + super().encode(value).to_bytes()) + + Value.to_codec_def().impl(Prefix) + assert Value(258).to_jam_bytes().to_bytes() == b'\xaa\x02\x01' + + +@pytest.mark.parametrize('hook', ['deserialize', 'serialize', 'encode']) +def test_builtin_instance_hooks_are_preserved(hook): + codec = UnsignedInteger(8) + value = model(codec, 2) + assert value.to_jam_bytes().to_bytes() == b'\x02' + calls = [] + original = getattr(codec, hook) + + def custom(self, item): + calls.append(hook) + return original(item) + + setattr(codec, hook, MethodType(custom, codec)) + expected = legacy_encode(value) + calls.clear() + assert value.to_jam_bytes() == expected + assert calls == [hook] + + +def test_class_serialization_hooks_preserve_validation(): + original = UnsignedInteger.serialize + + def checked(self, value): + if value == 255: + raise ScaleDeserializeException('class validation') + return original(self, value) + + with patch.object(UnsignedInteger, 'serialize', checked): + assert_matches_legacy(model(Vec(U16), [1, 255])) + + +def test_byte_vectors_preserve_indirect_bytes_hooks(): + def reject(self, value): + raise ScaleDeserializeException('indirect byte validation') + + with patch.object(Bytes, 'serialize', MethodType(reject, Bytes)): + assert_matches_legacy(model(BoundedVec(U8, 4), b'ab')) + + +def test_custom_iterable_is_consumed_once(): + for use_generator in (False, True): + calls = [] + + class Values(list): + def __iter__(self): + calls.append('iterate') + return super().__iter__() + + def generate(): + calls.append('iterate') + yield 1 + yield 258 + + value = model(Vec(U16), generate() if use_generator else Values([1, 258])) + assert value.to_jam_bytes().to_bytes() == b'\x02\x01\x00\x02\x01' + assert calls == ['iterate'] + + +def test_fallback_precedes_custom_attribute_access_and_schema_mutation(): + def make(): + calls = [] + payload_codec = BytesDef() + encode = payload_codec.encode + + @dataclass + class Nested(Serializable): + number: int = field(metadata={'codec': U8}) + + def read_number(self): + calls.append('get_number') + + def require_hex(self, value): + assert type(value) is str + return encode(value) + + payload_codec.encode = MethodType(require_hex, payload_codec) + return self.__dict__['number'] + + value = Nested(7) + Nested.number = property(read_number) + + @dataclass + class Outer(Serializable): + payload: bytes = field(metadata={'codec': payload_codec}) + nested: Nested = field(metadata={'codec': Nested.to_codec_def()}) + + return Outer(b'abc', value), calls + + old, old_calls = make() + expected = legacy_encode(old) + new, new_calls = make() + assert new.to_jam_bytes() == expected + assert old_calls == new_calls == ['get_number'] + + +def test_custom_truthiness_is_not_invoked_during_fast_path_probing(): + calls = [] + + class Integer(int): + def __bool__(self): + calls.append('bool') + return True + + value = model(U8, Integer(7)) + with pytest.raises(ValueError, match='Value must be an integer'): + value.to_jam_bytes() + assert calls == ['bool'] + + +def test_custom_truthiness_in_a_collection_is_invoked_once(): + calls = [] + + class Integer(int): + def __bool__(self): + calls.append('bool') + return True + + value = model(Vec(U16), [Integer(7)]) + with pytest.raises(ValueError, match='Value must be an integer'): + value.to_jam_bytes() + assert calls == ['bool'] + + +def test_direct_encoding_does_not_mutate_input_collections_or_buffers(): + @dataclass + class Collection(Serializable): + numbers: list = field(metadata={'codec': Array(U16, 2)}) + hashes: list = field(metadata={'codec': Vec(H256)}) + lookup: dict = field(metadata={'codec': Map(Bytes, Bytes)}) + + numbers = [1, 258] + digest = bytearray(range(32)) + hashes = [digest] + lookup = {b'z': b'last', b'a': b'first'} + value = Collection(numbers, hashes, lookup) + first = value.to_jam_bytes() + assert first == legacy_encode(value) + assert value.numbers is numbers and numbers == [1, 258] + assert value.hashes is hashes and hashes[0] is digest and digest == bytearray(range(32)) + assert value.lookup is lookup and list(lookup) == [b'z', b'a'] + numbers[0] = 2 + digest[0] = 255 + assert first != value.to_jam_bytes() + + +@pytest.mark.parametrize('kind', ['array', 'vec', 'bounded', 'map']) +def test_empty_collections_do_not_construct_elements(kind): + class NoElement(UnsignedInteger): + def new(self, **kwargs): + raise AssertionError('An empty collection must not construct elements') + + child = NoElement(8) + codec = { + 'array': Array(child, 0), + 'vec': Vec(child), + 'bounded': BoundedVec(child, 0), + 'map': Map(child, child), + }[kind] + value = model(codec, {} if kind == 'map' else []) + assert encode_from_object(value.to_codec_def(), value) == legacy_encode(value)