diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9c7015360..f5426b5de 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,6 +1,8 @@ { "files": { - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110" + "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", + "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", + "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14" }, "version": 1 } diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py new file mode 100644 index 000000000..aacb742b3 --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from smithy_core._dynamic_models import ModelRegistry +from smithy_core.exceptions import ModeledError + +from ._private import schemas + + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + + +_MODEL_INDEX = { + "GetExampleInput": "GET_EXAMPLE_INPUT", + "GetExampleOutput": "GET_EXAMPLE_OUTPUT", + "ExampleError": "EXAMPLE_ERROR", + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", + "ExampleValueText": "EXAMPLE_VALUE", + "ExampleValueNumber": "EXAMPLE_VALUE", + "ExampleValueEmpty": "EXAMPLE_VALUE", + "ExampleValueUnknown": "EXAMPLE_VALUE", + "Status": "STATUS", + "Code": "CODE", +} + +_RENAMES = { + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", +} + +_TARGET_NAMES = { + "example#Example": "RenamedExample", + "example#Value": "ExampleValue", +} + +_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") +__all__ = ("ServiceError", *_MODEL_NAMES) + + +_registry = ModelRegistry( + schemas, + renames=_RENAMES, + index=_MODEL_INDEX, + target_names=_TARGET_NAMES, + shape_names=frozenset( + { + "Code", + "ExampleError", + "ExampleValue", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "Status", + "Unit", + } + ), + service_error="ServiceError", +) +del schemas, ModelRegistry, ModeledError + + +def __getattr__(name: str): + value = _registry.build(name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi new file mode 100644 index 000000000..a4ba94f5e --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Self + +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import ModeledError +from smithy_core.response import ResponseMetadata +from smithy_core.serializers import ShapeSerializer +from smithy_core.types import UnknownEnumMixin + +__all__ = [ + "Code", + "ExampleError", + "ExampleValue", + "ExampleValueEmpty", + "ExampleValueNumber", + "ExampleValueText", + "ExampleValueUnknown", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "ServiceError", + "Status", + "Unit", +] + +class _SerdeShape: + def serialize(self, serializer: ShapeSerializer) -> None: ... + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: ... + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + +@dataclass(kw_only=True) +class GetExampleInput(_SerdeShape): + "Input for retrieving an example." + + id: str | None = ... + +@dataclass(kw_only=True) +class GetExampleOutput(_SerdeShape): + "Dataclass for GetExampleOutput structure." + + example: RenamedExample | None = ... + value: ExampleValue | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class ExampleError(ServiceError, _SerdeShape): + "An example modeled error." + + retry_after_seconds: int | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class RenamedExample(_SerdeShape): + "A representative generated structure." + + name: str + "The example name." + count: int | None = ... + tags: list[str] | None = ... + attributes: dict[str, Document] | None = ... + choice: ExampleValue | None = ... + status: str | None = ... + code: int | None = ... + +@dataclass +class ExampleValueText(_SerdeShape): + value: str + +@dataclass +class ExampleValueNumber(_SerdeShape): + value: int + +@dataclass +class ExampleValueEmpty(_SerdeShape): + value: Unit + +@dataclass +class ExampleValueUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +ExampleValue = ( + ExampleValueText | ExampleValueNumber | ExampleValueEmpty | ExampleValueUnknown +) +"A representative generated union." + +class _ExampleValueDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> ExampleValue: ... + +class Status(UnknownEnumMixin, StrEnum): + ACTIVE = "active" + INACTIVE = "inactive" + +class Code(UnknownEnumMixin, IntEnum): + OK = 200 + NOT_FOUND = 404 + +@dataclass(kw_only=True) +class Unit(_SerdeShape): + pass diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index 62b5b12e4..13e5883fc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -147,6 +147,9 @@ public String toMemberName(MemberShape shape) { } var memberName = escaper.escapeMemberName(CaseUtils.toSnakeCase(shape.getMemberName())); + if (memberName.startsWith("__") && !memberName.endsWith("__")) { + memberName += "___"; + } // Escape words that are only reserved for error members. if (shape.hasTrait(ErrorTrait.class)) { @@ -170,6 +173,16 @@ && carriesResponseMetadata(container)) { } if (container.isEnumShape() || container.isIntEnumShape()) { + if (memberName.startsWith("__") + && memberName.endsWith("__") + && !memberName.endsWith("___")) { + memberName += "_"; + } else if (memberName.startsWith("_") + && !memberName.startsWith("__") + && memberName.endsWith("_") + && !memberName.endsWith("__")) { + memberName += "_"; + } memberName = memberName.toUpperCase(Locale.ENGLISH); } return memberName; diff --git a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt index 1831fb341..b18e8e1a8 100644 --- a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt +++ b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -81,3 +81,17 @@ str # that are likely imported in the generated code (e.g. datetime) # We only escape the types we use. datetime + +# Generated model runtime methods. +serialize +serialize_members +deserialize + +# Dataclass-generated methods and hooks. +__init__ +__post_init__ +__repr__ +__annotations__ + +# Generated model runtime methods. +_smithy_default diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java index 0102a560c..3941b92e2 100644 --- a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java @@ -137,6 +137,68 @@ public void testUnrelatedMembersOnOutputsAndErrorsAreUnaffected() { assertEquals("retry_after", memberName(provider, model, "ThingError$retryAfter")); } + @Test + public void testDataclassHooksAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + operations: [GetThing] + } + + operation GetThing { + input: GetThingInput + } + + structure GetThingInput { + __init__: String + __post_init__: String + _smithy_default: String + __repr__: String + __annotations__: String + __value: String + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("__init___", memberName(provider, model, "GetThingInput$__init__")); + assertEquals("__post_init___", memberName(provider, model, "GetThingInput$__post_init__")); + assertEquals("_smithy_default_", memberName(provider, model, "GetThingInput$_smithy_default")); + assertEquals("__repr___", memberName(provider, model, "GetThingInput$__repr__")); + assertEquals("__annotations___", memberName(provider, model, "GetThingInput$__annotations__")); + assertEquals("__value___", memberName(provider, model, "GetThingInput$__value")); + } + + @Test + public void testEnumSunderNamesAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + } + + enum StringValues { + _VALUE_ = "value" + __DUNDER__ = "dunder" + } + + intEnum IntValues { + _VALUE_ = 1 + __DUNDER__ = 2 + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("_VALUE__", memberName(provider, model, "StringValues$_VALUE_")); + assertEquals("_VALUE__", memberName(provider, model, "IntValues$_VALUE_")); + assertEquals("__DUNDER___", memberName(provider, model, "StringValues$__DUNDER__")); + assertEquals("__DUNDER___", memberName(provider, model, "IntValues$__DUNDER__")); + } + private static final String RESPONSE_METADATA_MODEL = """ $version: "2" namespace smithy.example diff --git a/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json new file mode 100644 index 000000000..723b0ddcb --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json @@ -0,0 +1,4 @@ +{ + "type": "enhancement", + "description": "Added optional full shape ID target mappings to `ModelRegistry` so generators can resolve renamed and same-named cross-namespace model targets without ambiguous local-name lookups." +} diff --git a/packages/smithy-core/src/smithy_core/_dynamic_models.py b/packages/smithy-core/src/smithy_core/_dynamic_models.py index 028323346..1ee10c042 100644 --- a/packages/smithy-core/src/smithy_core/_dynamic_models.py +++ b/packages/smithy-core/src/smithy_core/_dynamic_models.py @@ -18,7 +18,7 @@ import datetime from base64 import b64decode -from collections.abc import Collection +from collections.abc import Collection, Mapping from dataclasses import field as _field from dataclasses import make_dataclass from decimal import Decimal @@ -30,7 +30,7 @@ from ._model_naming import member_field_name, union_unknown_name, variant_class_name from ._schema_compact import hydrate from .deserializers import ShapeDeserializer -from .exceptions import ModeledError +from .exceptions import ModeledError, SerializationError from .schemas import Schema, SchemaExtension from .serializers import ShapeSerializer from .shapes import ShapeID, ShapeType @@ -151,14 +151,15 @@ class ModelRegistry: def __init__( self, schemas_module: ModuleType, - renames: dict[str, str] | None = None, + renames: Mapping[str, str] | None = None, *, - index: dict[str, str] | None = None, + index: Mapping[str, str] | None = None, + target_names: Mapping[str, str] | None = None, shape_names: Collection[str] | None = None, service_error: str | None = None, ) -> None: self._schemas = schemas_module - self._renames = renames or {} + self._renames = dict(renames or {}) self._classes: dict[str, type] = {} self._union_schemas: dict[str, Schema] = {} # Reentrant because build() recurses: unions build their variants, and a @@ -169,13 +170,22 @@ def __init__( # A codegen-supplied index (name -> schema-const, variant/unknown names already # disambiguated) lets the registry resolve a build() without hydrating every # schema in the module just to map names; the scanning fallback hydrates all. - self._index: dict[str, str] | None = index + self._index = dict(index) if index is not None else None + # Full shape ID -> generated public model name. Full IDs keep member-target + # resolution unambiguous when service renames distinguish same-named shapes + # from different namespaces. + self._target_names = dict(target_names or {}) + self._renamed_schemas = { + schema_name: model_name for model_name, schema_name in self._renames.items() + } self._shape_names: set[str] = set(shape_names) if shape_names else set() # Generated clients name their per-service error base (a ModeledError subclass # emitted into the models module). Modeled errors extend it so `except # ServiceError` catches them; a hand-built registry omits it and falls back to # ModeledError. self._service_error_name = service_error + if service_error is not None: + self._shape_names.add(service_error) self._codec = SchemaExtension(self._resolve_member_codec) self._field_names_ext = SchemaExtension(self._compute_field_names) @@ -230,7 +240,7 @@ def build(self, name: str) -> type: # Event-stream union deserializer helper: _Deserializer().deserialize(d). if name.startswith("_") and name.endswith("Deserializer"): union_name = name[1 : -len("Deserializer")] - if union_name in index: + if union_name in index or union_name in self._renames: built = self._build_union_deserializer(union_name) self._classes[name] = built return built @@ -253,14 +263,26 @@ def build(self, name: str) -> type: return built raise AttributeError(name) schema = hydrate(getattr(self._schemas, const)) + canonical = self._renamed_schemas.get(const) if schema.shape_type == ShapeType.UNION: - self._build_union(schema.id.name, schema) - return self._classes[name] + self._build_union(canonical or schema.id.name, schema) + built = self._classes.get(name) + if built is None and name == schema.id.name: + built = self._classes[canonical or schema.id.name] + if built is None: + raise AttributeError(name) + self._classes[name] = built + return built + canonical = canonical or name + if cached := self._classes.get(canonical): + self._classes[name] = cached + return cached if schema.shape_type in (ShapeType.ENUM, ShapeType.INT_ENUM): - built = self._build_enum(name, schema) + built = self._build_enum(canonical, schema) else: - built = self._build_struct(name, schema) + built = self._build_struct(canonical, schema) built.__module__ = module + self._classes[canonical] = built self._classes[name] = built return built @@ -323,15 +345,16 @@ def _error_correct( def _correction_default(self, member: Schema) -> Any: st = member.shape_type if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) return self.build(target)._smithy_default() # type: ignore[attr-defined] if st == ShapeType.UNION: - target = member.expect_member_target().id.name - return self.build(union_unknown_name(target, self._shape_names))(tag="") + target = self._target_name(member.expect_member_target()) + return self.build(self._union_unknown_name(target))(tag="") return _CORRECTION_SCALARS[st]() def _build_struct(self, name: str, schema: Schema) -> type: - is_error = schema.get_trait(ErrorTrait) is not None + error_trait = schema.get_trait(ErrorTrait) + is_error = error_trait is not None # Per Smithy @input structure constraints, every member of an input # structure is effectively optional regardless of @required; only # non-input structures make a @required, non-@default member mandatory. @@ -339,11 +362,25 @@ def _build_struct(self, name: str, schema: Schema) -> type: field_names = self._field_names(schema) fields: list[Any] = [] for i, member in enumerate(schema.members_by_index): + if is_error and field_names[i] == "message": + if member.get_trait(SensitiveTrait) is not None: + fields.append( + ( + "message", + str, + _field(default="", kw_only=False, repr=False), + ) + ) + continue spec = self._field_spec(member, input_nullable) if spec is _MANDATORY: fields.append((field_names[i], _py_type(member))) else: fields.append((field_names[i], _py_type(member), spec)) + if error_trait is not None: + fields.append( + ("fault", str | None, _field(default=error_trait.fault.value)) + ) bases: tuple[type, ...] = (self._error_base(),) if is_error else () # Operation outputs carry a non-modeled response_metadata attribute, excluded # from eq and repr. Errors inherit it from ModeledError instead. @@ -456,16 +493,32 @@ def _default_factory(self, member: Schema, trait: DefaultTrait) -> Any: return None def _build_union(self, name: str, schema: Schema) -> None: + if name in self._union_schemas: + return module = self._models_module_name for member in schema.members_by_index: - vname = variant_class_name( - name, member.expect_member_name(), self._shape_names - ) + vname = self._union_variant_name(name, member.expect_member_name()) variant = self._build_variant(vname, schema, member.expect_member_index()) variant.__module__ = module self._classes[vname] = variant - unknown = union_unknown_name(name, self._shape_names) - unknown_cls = make_dataclass(unknown, [("tag", str)]) + unknown = self._union_unknown_name(name) + + def reject_serialization(self: Any, serializer: ShapeSerializer) -> None: + raise SerializationError("Unknown union variants may not be serialized.") + + @classmethod # type: ignore[misc] + def deserialize(cls: type, deserializer: ShapeDeserializer) -> Any: + raise NotImplementedError + + unknown_cls = make_dataclass( + unknown, + [("tag", str)], + namespace={ + "serialize": reject_serialization, + "serialize_members": reject_serialization, + "deserialize": deserialize, + }, + ) unknown_cls.__module__ = module self._classes[unknown] = unknown_cls self._union_schemas[name] = schema @@ -474,6 +527,55 @@ def _build_union(self, name: str, schema: Schema) -> None: marker = make_dataclass(name, []) marker.__module__ = module self._classes[name] = marker + original = schema.id.name + index = self._ensure_index() + const = self._renames.get(name, index.get(name)) + if ( + original != name + and original not in self._renames + and const is not None + and index.get(original) == const + ): + self._classes.setdefault(original, marker) + self._union_schemas.setdefault(original, schema) + for member in schema.members_by_index: + member_name = member.expect_member_name() + canonical_variant = self._union_variant_name(name, member_name) + original_variant = self._union_variant_name(original, member_name) + if ( + original_variant not in self._renames + and index.get(original_variant, const) == const + ): + self._classes.setdefault( + original_variant, self._classes[canonical_variant] + ) + original_unknown = self._union_unknown_name(original) + if ( + original_unknown not in self._renames + and index.get(original_unknown, const) == const + ): + self._classes.setdefault(original_unknown, unknown_cls) + + def _union_variant_name(self, container: str, member: str) -> str: + capitalized = member[:1].upper() + member[1:] + return self._union_synthetic_name(container, capitalized) + + def _union_unknown_name(self, container: str) -> str: + return self._union_synthetic_name(container, "Unknown") + + def _union_synthetic_name(self, container: str, suffix: str) -> str: + base = container + suffix + disambiguated = container + "_" + suffix + index = self._ensure_index() + const = self._renames.get(container, index.get(container)) + if const is not None: + for candidate in (base, disambiguated): + if self._renames.get(candidate, index.get(candidate)) == const: + return candidate + owner = self._renames.get(base, index.get(base)) + if owner is not None and owner != const: + return disambiguated + return disambiguated if base in self._shape_names else base def _build_variant(self, vname: str, union_schema: Schema, index: int) -> type: registry = self @@ -539,15 +641,13 @@ def consume(member_schema: Schema, de: ShapeDeserializer) -> None: member = by_index[member_schema.expect_member_index()] _, reader = member.get_extension(self._codec) variant = self.build( - variant_class_name(name, member.expect_member_name(), self._shape_names) + self._union_variant_name(name, member.expect_member_name()) ) result[0] = variant(value=reader(de, member)) deserializer.read_struct(schema, consumer=consume) if result[0] is None: - result[0] = self.build(union_unknown_name(name, self._shape_names))( - tag="unknown" - ) + result[0] = self.build(self._union_unknown_name(name))(tag="unknown") return result[0] def _is_streaming(self, member: Schema) -> bool: @@ -566,7 +666,7 @@ def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: # coerce the raw primitive back through the enum class (matching the eager # EnumGenerator's `EnumClass(read_string(...))`). is_int = st == ShapeType.INT_ENUM - enum_cls = self.build(member.expect_member_target().id.name) + enum_cls = self.build(self._target_name(member.expect_member_target())) write_name = "write_integer" if is_int else "write_string" read_name = "read_integer" if is_int else "read_string" @@ -581,7 +681,7 @@ def re(d: ShapeDeserializer, m: Schema) -> Any: if scalar is not None: return scalar if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) def wr(s: ShapeSerializer, m: Schema, v: Any) -> None: s.write_struct(m, v) @@ -597,7 +697,7 @@ def rr(d: ShapeDeserializer, m: Schema) -> Any: return wr, rr if st == ShapeType.UNION: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) self.build(target) # ensure union schema is registered def wu(s: ShapeSerializer, m: Schema, v: Any) -> None: @@ -619,6 +719,17 @@ def ru(d: ShapeDeserializer, m: Schema) -> Any: ) raise NotImplementedError(f"no codec for shape type {st}") + def _target_name(self, schema: Schema) -> str: + if generated := self._target_names.get(str(schema.id)): + return generated + index = self._ensure_index() + const = index.get(schema.id.name) + return ( + self._renamed_schemas.get(const, schema.id.name) + if const is not None + else schema.id.name + ) + def _list_codec(self, elem: Schema, sparse: bool) -> tuple[_Writer, _Reader]: def w(s: ShapeSerializer, m: Schema, v: list[Any]) -> None: ew, _ = elem.get_extension(self._codec) diff --git a/packages/smithy-core/src/smithy_core/_model_naming.py b/packages/smithy-core/src/smithy_core/_model_naming.py index 05245e2e8..1fb098e3f 100644 --- a/packages/smithy-core/src/smithy_core/_model_naming.py +++ b/packages/smithy-core/src/smithy_core/_model_naming.py @@ -18,7 +18,8 @@ """and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield bool bytes bytearray dict float int list str - datetime""".split() + datetime serialize serialize_members deserialize __init__ __post_init__ + _smithy_default __repr__ __annotations__""".split() ) # reserved-error-member-names.txt _RESERVED_ERROR_MEMBERS = frozenset( @@ -54,14 +55,24 @@ def member_field_name( return "message" name = to_snake_case(wire_name) - if name in _RESERVED_MEMBERS: + if name.startswith("__") and not name.endswith("__"): + name += "___" + elif name in _RESERVED_MEMBERS: name += "_" if is_error and name in _RESERVED_ERROR_MEMBERS: name += "_" if name == _RESPONSE_METADATA_MEMBER and carries_response_metadata: name += "_" if is_enum: - name = name.upper() + name = _escape_enum_underscores(name).upper() + return name + + +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" return name diff --git a/packages/smithy-core/tests/unit/test_dynamic_models.py b/packages/smithy-core/tests/unit/test_dynamic_models.py index 60d4363dc..6f342ca03 100644 --- a/packages/smithy-core/tests/unit/test_dynamic_models.py +++ b/packages/smithy-core/tests/unit/test_dynamic_models.py @@ -3,19 +3,40 @@ # pyright: reportPrivateUsage=false from enum import IntEnum, StrEnum from types import ModuleType -from typing import Any, cast +from typing import Any, Protocol, cast +import pytest from smithy_core import _schema_compact from smithy_core._dynamic_models import ModelRegistry from smithy_core._schema_compact import StaticSchema from smithy_core.deserializers import ShapeDeserializer -from smithy_core.exceptions import ModeledError +from smithy_core.exceptions import ModeledError, SerializationError +from smithy_core.schemas import Schema +from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeType from smithy_core.types import UnknownEnumMixin NS = "ns.example" +class _UnknownVariant(Protocol): + def serialize(self, serializer: ShapeSerializer) -> None: ... + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _UnknownVariantClass(Protocol): + def __call__(self, *, tag: str) -> _UnknownVariant: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _UnknownVariant: ... + + +class _ModeledErrorClass(Protocol): + def __call__(self) -> ModeledError: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> ModeledError: ... + + def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: return ( ShapeType.STRUCTURE.value, @@ -30,12 +51,17 @@ def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: _ERROR_TRAIT = "smithy.api#error" -def _error_struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: +def _error_struct( + name: str, + members: tuple[tuple[str, object], ...], + *, + fault: str = "client", +) -> StaticSchema: return ( ShapeType.STRUCTURE.value, NS, name, - {_ERROR_TRAIT: "client"}, + {_ERROR_TRAIT: fault}, tuple(m[0] for m in members), tuple((m[1], 0) for m in members), ) @@ -128,6 +154,249 @@ def test_supplied_index_matches_scanning_fallback() -> None: assert _fields(scanned) == _fields(supplied) +def test_supplied_index_preserves_distinct_same_named_structures() -> None: + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ("first",), + ((ShapeType.STRING.value, 0),), + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("second",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second), + index={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + ) + + first_cls = registry.build("FirstThing") + second_cls = registry.build("SecondThing") + assert first_cls is not second_cls + assert first_cls.__name__ == "FirstThing" + assert second_cls.__name__ == "SecondThing" + assert _fields(first_cls) == ["first"] + assert _fields(second_cls) == ["second"] + + +def test_registry_copies_supplied_name_maps() -> None: + renames = {"Alias": "FOO"} + index = {"Alias": "FOO"} + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames=renames, + index=index, + ) + renames.clear() + index.clear() + + assert registry.build("Alias").__name__ == "Alias" + + +def test_alias_and_canonical_struct_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + + +def test_alias_and_canonical_union_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_union("Foo", (("value", ShapeType.STRING.value),))), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + variant = registry.build("AliasValue") + unknown = registry.build("AliasUnknown") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + assert registry.build("AliasValue") is variant + assert registry.build("FooValue") is variant + assert registry.build("FooUnknown") is unknown + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + consumer(schema.members_by_index[0], self) + + def read_string(self, schema: Schema) -> str: + return "value" + + alias_deserializer = registry.build("_AliasDeserializer")() + original_deserializer = registry.build("_FooDeserializer")() + alias_value = alias_deserializer.deserialize(cast(ShapeDeserializer, _De())) + original_value = original_deserializer.deserialize(cast(ShapeDeserializer, _De())) + assert type(alias_value) is variant + assert type(original_value) is variant + assert alias_value.value == original_value.value == "value" + + +def test_renamed_union_does_not_replace_same_named_shape() -> None: + renamed = ( + ShapeType.UNION.value, + "a.example", + "Thing", + 0, + ("value",), + ((ShapeType.STRING.value, 0),), + ) + other = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("other",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(A_OTHER=other, Z_RENAMED=renamed), + renames={"FirstThing": "Z_RENAMED"}, + ) + + first = registry.build("FirstThing") + thing = registry.build("Thing") + assert first is not thing + assert _fields(thing) == ["other"] + + +def test_supplied_union_names_override_original_shape_names() -> None: + union = ( + ShapeType.UNION.value, + "a.example", + "Foo", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + item = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldItem", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + unknown = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldUnknown", + 0, + ("unknown",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FOO=union, ITEM=item, UNKNOWN=unknown), + renames={ + "Choice": "FOO", + "ChoiceItem": "ITEM", + "ChoiceUnknown": "UNKNOWN", + }, + index={ + "Foo": "FOO", + "Choice_Item": "FOO", + "Choice_Unknown": "FOO", + "OldItem": "ITEM", + "OldUnknown": "UNKNOWN", + }, + shape_names={"Foo", "OldItem", "OldUnknown"}, + ) + + item_cls = registry.build("ChoiceItem") + unknown_cls = registry.build("ChoiceUnknown") + registry.build("Choice") + assert registry.build("Choice_Item") is not item_cls + assert registry.build("Choice_Unknown") is not unknown_cls + assert registry.build("ChoiceItem") is item_cls + assert registry.build("ChoiceUnknown") is unknown_cls + + +def test_renamed_union_aliases_do_not_replace_renamed_models() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + VALUE=_struct("ValueModel", (("value", ShapeType.STRING.value),)), + UNKNOWN=_struct("UnknownModel", (("value", ShapeType.STRING.value),)), + ), + renames={ + "Alias": "FOO", + "FooValue": "VALUE", + "FooUnknown": "UNKNOWN", + }, + ) + + registry.build("Alias") + assert _fields(registry.build("FooValue")) == ["value"] + assert _fields(registry.build("FooUnknown")) == ["value"] + assert registry.build("FooValue") is not registry.build("AliasValue") + assert registry.build("FooUnknown") is not registry.build("AliasUnknown") + + +def test_renamed_union_original_alias_does_not_replace_renamed_model() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + MODEL=_struct("Model", (("value", ShapeType.STRING.value),)), + ), + renames={"Alias": "FOO", "Foo": "MODEL"}, + ) + + registry.build("Alias") + assert _fields(registry.build("Foo")) == ["value"] + assert registry.build("Foo") is not registry.build("Alias") + + +def test_target_names_resolve_renames_by_full_shape_id() -> None: + _schema_compact._cache.clear() + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ) + container = _struct("Container", (("value", lambda: second),)) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second, CONTAINER=container), + renames={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + index={ + "FirstThing": "FIRST", + "SecondThing": "SECOND", + "Container": "CONTAINER", + }, + target_names={ + "a.example#Thing": "FirstThing", + "b.example#Thing": "SecondThing", + f"{NS}#Container": "Container", + }, + shape_names={"FirstThing", "SecondThing", "Container"}, + ) + schema = _schema_compact.hydrate(container) + member = schema.members_by_index[0] + _, reader = member.get_extension(registry._codec) + + class _De: + def read_struct(self, schema: object, consumer: object) -> None: + pass + + result = reader(cast(ShapeDeserializer, _De()), member) + assert type(result).__name__ == "SecondThing" + + def test_supplied_index_disambiguated_union_variant() -> None: # Union FooWarning.message collides with struct FooWarningMessage; codegen emits # the "_"-disambiguated variant name in the index. The real struct must still @@ -220,6 +489,20 @@ def test_modeled_error_falls_back_to_modeled_error_without_service_error() -> No assert cls.__mro__[1] is ModeledError +def test_modeled_error_uses_modeled_fault() -> None: + _schema_compact._cache.clear() + mod = _module(BOOM=_error_struct("Boom", (), fault="server")) + registry = ModelRegistry(mod, index={"Boom": "BOOM"}, shape_names={"Boom"}) + cls = cast(_ModeledErrorClass, registry.build("Boom")) + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + pass + + assert cls().fault == "server" + assert cls.deserialize(cast(ShapeDeserializer, _De())).fault == "server" + + def test_enum_member_deserialize_coerces_to_enum_class() -> None: # A member whose target is an enum must deserialize to the enum class, not the # raw wire string (the eager generator wrapped it as EnumClass(read_string(...))). @@ -265,6 +548,22 @@ def test_union_classes_report_models_module() -> None: assert registry.build(name).__module__ == "ns.example.models" +def test_unknown_union_variant_rejects_serde() -> None: + _schema_compact._cache.clear() + mod = _module(FOO=_union("Foo", (("a", ShapeType.STRING.value),))) + registry = ModelRegistry(mod, index={"Foo": "FOO"}, shape_names={"Foo"}) + registry.build("Foo") + unknown = cast(_UnknownVariantClass, registry.build("FooUnknown")) + value = unknown(tag="future") + + with pytest.raises(SerializationError): + value.serialize(cast(ShapeSerializer, object())) + with pytest.raises(SerializationError): + value.serialize_members(cast(ShapeSerializer, object())) + with pytest.raises(NotImplementedError): + unknown.deserialize(cast(ShapeDeserializer, object())) + + def test_concurrent_build_returns_one_class() -> None: import threading diff --git a/packages/smithy-core/tests/unit/test_model_naming.py b/packages/smithy-core/tests/unit/test_model_naming.py index d47d7e495..d0198c905 100644 --- a/packages/smithy-core/tests/unit/test_model_naming.py +++ b/packages/smithy-core/tests/unit/test_model_naming.py @@ -1,6 +1,89 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from smithy_core._model_naming import union_unknown_name, variant_class_name +from smithy_core._model_naming import ( + member_field_name, + to_snake_case, + union_unknown_name, + variant_class_name, +) + + +def test_snake_case_preserves_prefix_for_leading_digit() -> None: + assert to_snake_case("_1") == "_1" + assert to_snake_case("_2HTTPServer") == "_2_http_server" + + +def test_snake_case_preserves_existing_underscores() -> None: + assert to_snake_case("_value") == "_value" + assert to_snake_case("value_") == "value_" + assert to_snake_case("some__value") == "some__value" + + +def test_enum_names_escape_before_uppercasing() -> None: + assert ( + member_field_name( + "CLASS", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "CLASS_" + ) + + +def test_enum_sunder_names_are_escaped() -> None: + assert ( + member_field_name( + "_VALUE_", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "_VALUE__" + ) + + +def test_enum_dunder_names_are_escaped() -> None: + assert ( + member_field_name( + "__VALUE__", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "__VALUE___" + ) + + +def test_runtime_method_names_are_reserved() -> None: + for name in ( + "serialize", + "serializeMembers", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + ): + assert member_field_name( + name, + is_error=False, + carries_response_metadata=False, + is_enum=False, + ).endswith("_") + + +def test_double_leading_underscore_names_avoid_name_mangling() -> None: + assert ( + member_field_name( + "__value", + is_error=False, + carries_response_metadata=False, + is_enum=False, + ) + == "__value___" + ) def test_variant_class_name_default() -> None: diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json new file mode 100644 index 000000000..d5c28cdf5 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added native generation of lazy runtime model modules and complete static model stubs for standalone types packages." +} diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py index 736f71821..79fa17069 100644 --- a/packages/smithy-python/src/smithy_python/files.py +++ b/packages/smithy-python/src/smithy_python/files.py @@ -53,7 +53,11 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: raise _exceptions.CodegenError( f"Generated path {path!r} is already a text file" ) - writer = _writer.PythonWriter(module, dependencies=self._dependencies) + writer = _writer.PythonWriter( + module, + dependencies=self._dependencies, + stub=path.endswith(".pyi"), + ) self._files[path] = _PythonFile(module, writer) return writer diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index 3bec23e70..9d2b4ece7 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -25,6 +25,9 @@ "TypesSettings", ] +_SERVICE_ERROR = "ServiceError" +_UNIT = _model.ShapeId("smithy.api", "Unit") + @dataclass(frozen=True, slots=True) class ArtifactSettings: @@ -47,10 +50,15 @@ class TypesSettings(ArtifactSettings): class ModelPlan: """Derived semantic data used to emit lazy model declarations.""" + shapes: tuple[_model.Shape, ...] declaration_names: Mapping[_model.ShapeId, str] renames: Mapping[str, str] union_variant_names: Mapping[_model.ShapeId, str] union_unknown_names: Mapping[_model.ShapeId, str] + registry_index: Mapping[str, str] + target_names: Mapping[str, str] + shape_names: frozenset[str] + includes_unit: bool @classmethod def from_shapes( @@ -60,31 +68,55 @@ def from_shapes( schema_names: Mapping[_model.ShapeId, str], ) -> ModelPlan: """Plan declaration and union names in stable shape order.""" + model_shapes: list[_model.Shape] = [] declarations: dict[_model.ShapeId, str] = {} variants: dict[_model.ShapeId, str] = {} unknowns: dict[_model.ShapeId, str] = {} + index: dict[str, str] = {} + renames: dict[str, str] = {} + target_names: dict[str, str] = {} + includes_unit = any( + shape.type not in (_model.ShapeType.ENUM, _model.ShapeType.INT_ENUM) + and any(member.target == _UNIT for member in shape.members.values()) + for shape in shapes + ) for shape in shapes: if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( "streaming" ): continue - declarations[shape.id] = symbols.declaration_name(shape.id) + model_shapes.append(shape) + declaration = symbols.declaration_name(shape.id) + schema = schema_names[shape.id] + declarations[shape.id] = declaration + index[declaration] = schema + if declaration != shape.id.name: + renames[declaration] = schema + target_names[str(shape.id)] = declaration if shape.type is _model.ShapeType.UNION: for member in shape.members.values(): - variants[member.id] = symbols.union_variant_name(member.id) - unknowns[shape.id] = symbols.union_unknown_name(shape.id) + variant = symbols.union_variant_name(member.id) + variants[member.id] = variant + index[variant] = schema + unknown = symbols.union_unknown_name(shape.id) + unknowns[shape.id] = unknown + index[unknown] = schema return cls( + shapes=tuple(model_shapes), declaration_names=MappingProxyType(declarations), - renames=MappingProxyType( - { - declaration: schema_names[shape_id] - for shape_id, declaration in declarations.items() - if declaration != shape_id.name - } - ), + renames=MappingProxyType(renames), union_variant_names=MappingProxyType(variants), union_unknown_names=MappingProxyType(unknowns), + registry_index=MappingProxyType(index), + target_names=MappingProxyType(target_names), + shape_names=frozenset( + ( + *declarations.values(), + *(("Unit",) if includes_unit else ()), + ) + ), + includes_unit=includes_unit, ) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py new file mode 100644 index 000000000..14ac548dc --- /dev/null +++ b/packages/smithy-python/src/smithy_python/models.py @@ -0,0 +1,457 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Emission of lazy runtime models and their static typing surface.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping +from dataclasses import replace +from json import dumps +from textwrap import wrap +from typing import TYPE_CHECKING, Final + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import model as _model +from . import symbols as _symbols + +if TYPE_CHECKING: + from .generation import GenerationContext, ModelPlan, TypesSettings + from .writer import PythonWriter + +__all__ = ["ModelGenerator"] + + +class ModelGenerator: + """Emit lazy runtime models and the complete static model stub.""" + + def generate(self, context: GenerationContext[TypesSettings]) -> None: + """Register ``models.py`` and ``models.pyi`` with the file manifest.""" + _validate_member_names(context) + package = context.settings.package + directory = package.replace(".", "/") + _write_runtime( + context.files.python_module( + f"src/{directory}/models.py", f"{package}.models" + ), + context.plan.models, + ) + _StubEmitter( + context.files.python_module( + f"src/{directory}/models.pyi", f"{package}.models" + ), + context, + ).write() + + +def _core_reference(name: str, module: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, f"smithy_core.{module}", dependencies=(_dependencies.SMITHY_CORE,) + ) + + +_MODEL_REGISTRY: Final = _core_reference("ModelRegistry", "_dynamic_models") +_MODELED_ERROR: Final = _core_reference("ModeledError", "exceptions") +_SHAPE_SERIALIZER: Final = _core_reference("ShapeSerializer", "serializers") +_SHAPE_DESERIALIZER: Final = _core_reference("ShapeDeserializer", "deserializers") +_UNKNOWN_ENUM_MIXIN: Final = _core_reference("UnknownEnumMixin", "types") +_RESPONSE_METADATA: Final = _core_reference("ResponseMetadata", "response") +_DATACLASS: Final = _symbols.TypeReference("dataclass", "dataclasses") +_SELF: Final = _symbols.TypeReference("Self", "typing") +_STR_ENUM: Final = _symbols.TypeReference("StrEnum", "enum") +_INT_ENUM: Final = _symbols.TypeReference("IntEnum", "enum") +_SCHEMAS: Final = _symbols.TypeReference("schemas", "._private") + +_ERROR_MESSAGE_NAMES: Final = frozenset({"errormessage", "error_message", "message"}) +_DOCSTRING_WIDTH: Final = 72 +_MODEL_MEMBER_NAMES: Final = frozenset( + { + "bool", + "bytes", + "bytearray", + "dict", + "float", + "int", + "list", + "str", + "datetime", + "serialize", + "serialize_members", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + } +) +_ERROR_MEMBER_NAMES: Final = frozenset( + {"is_retry_safe", "retry_after", "is_throttling_error", "fault"} +) +_RESPONSE_METADATA_DOC: Final = ( + "Metadata about the response that produced this output.\n\n" + "Use this to recover the request identifiers a service's support team needs " + "in order to investigate a call. Members of the metadata are individually " + "optional." +) +_SERVICE_ERROR_DOC: Final = ( + "Base error for all errors in the service.\n\n" + "Some exceptions do not extend from this class, including synthetic, implicit, " + "and shared exception types." +) +_UNKNOWN_UNION_DOC: Final = ( + "Represents an unknown variant.\n\n" + "If you receive this value, you will need to update your library to receive " + "the parsed value.\n\n" + "This value may not be deliberately sent." +) + + +def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: + exports = _model_exports(plan) + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + writer.line() + writer.line() + _write_mapping(writer, "_MODEL_INDEX", plan.registry_index) + writer.line() + _write_mapping(writer, "_RENAMES", plan.renames) + writer.line() + _write_mapping(writer, "_TARGET_NAMES", plan.target_names) + writer.line() + additional = exports[len(plan.registry_index) + 1 :] + if additional: + writer.line( + "_MODEL_NAMES = (*tuple(_MODEL_INDEX), ", + ", ".join(map(_quote, additional)), + ")", + ) + else: + writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") + writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line() + writer.line() + writer.line("_registry = ", _MODEL_REGISTRY, "(") + with writer.indent(): + writer.line(_SCHEMAS, ",") + writer.line("renames=_RENAMES,") + writer.line("index=_MODEL_INDEX,") + writer.line("target_names=_TARGET_NAMES,") + _write_string_set(writer, "shape_names", plan.shape_names, suffix=",") + writer.line('service_error="ServiceError",') + writer.line(")") + writer.line("del ", _SCHEMAS, ", ", _MODEL_REGISTRY, ", ", _MODELED_ERROR) + writer.line() + writer.line() + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("value = _registry.build(name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *__all__})") + + +class _StubEmitter: + def __init__( + self, + writer: PythonWriter, + context: GenerationContext[TypesSettings], + ) -> None: + self.writer = writer + self.model = context.model + self.plan = context.plan.models + self.symbols = context.symbols + self.module = f"{context.settings.package}.models" + operations = tuple(context.model.iter_shapes(_model.OperationShape)) + self.inputs = {operation.input for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("input") + } + self.outputs = {operation.output for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("output") + } + + def write(self) -> None: + writer = self.writer + writer.line("__all__ = [") + with writer.indent(): + for name in sorted(_model_exports(self.plan)): + writer.line(_quote(name), ",") + writer.line("]") + writer.line() + writer.line("class _SerdeShape:") + with writer.indent(): + writer.line( + "def serialize(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line( + "def serialize_members(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line("@classmethod") + writer.line( + "def deserialize(cls, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _SELF, + ": ...", + ) + writer.line() + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + + for shape in self.plan.shapes: + writer.line() + match shape.type: + case _model.ShapeType.STRUCTURE: + self._write_structure(shape) + case _model.ShapeType.UNION: + self._write_union(shape) + case _model.ShapeType.ENUM | _model.ShapeType.INT_ENUM: + self._write_enum(shape) + case _: + pass + if self.plan.includes_unit: + writer.line() + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line("class Unit(_SerdeShape):") + with writer.indent(): + writer.line("pass") + + def _write_structure(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + error = shape.has_trait("error") + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line( + "class ", + name, + "(ServiceError, _SerdeShape):" if error else "(_SerdeShape):", + ) + with writer.indent(): + members = tuple( + member + for member in shape.members.values() + if not error or member.name.lower() not in _ERROR_MESSAGE_NAMES + ) + carries_metadata = error or shape.id in self.outputs + _write_docstring( + writer, + _documentation(shape) or f"Dataclass for {shape.id.name} structure.", + ) + if members or carries_metadata: + writer.line() + for member in members: + reference = self.symbols.type_reference(member.target) + has_default = member.has_trait("default") or self.model.get_target( + member + ).has_trait("default") + mandatory = ( + shape.id not in self.inputs + and member.has_trait("required") + and not has_default + ) + if not mandatory: + reference = replace(reference, nullable=True) + writer.line( + self.symbols.member_name(member.id), + ": ", + reference, + "" if mandatory else " = ...", + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if carries_metadata: + writer.line("response_metadata: ", _RESPONSE_METADATA, " = ...") + _write_docstring(writer, _RESPONSE_METADATA_DOC) + + def _write_union(self, shape: _model.Shape) -> None: + writer = self.writer + variants: list[_symbols.TypeReference] = [] + for member in shape.members.values(): + name = self.plan.union_variant_names[member.id] + variants.append(_symbols.TypeReference(name, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", name, "(_SerdeShape):") + with writer.indent(): + if docs := _documentation(member): + _write_docstring(writer, docs) + writer.line() + writer.line("value: ", self.symbols.type_reference(member.target)) + writer.line() + + unknown = self.plan.union_unknown_names[shape.id] + variants.append(_symbols.TypeReference(unknown, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", unknown, "(_SerdeShape):") + with writer.indent(): + _write_docstring(writer, _UNKNOWN_UNION_DOC) + writer.line() + writer.line("tag: str") + writer.line() + + name = self.plan.declaration_names[shape.id] + writer.line(name, " = ", *_join_types(variants)) + if docs := _documentation(shape): + _write_docstring(writer, docs) + writer.line() + writer.line("class _", name, "Deserializer:") + with writer.indent(): + writer.line( + "def deserialize(self, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _symbols.TypeReference(name, self.module), + ": ...", + ) + + def _write_enum(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + base = _INT_ENUM if shape.type is _model.ShapeType.INT_ENUM else _STR_ENUM + writer.line("class ", name, "(", _UNKNOWN_ENUM_MIXIN, ", ", base, "):") + with writer.indent(): + docs = _documentation(shape) + if docs: + _write_docstring(writer, docs) + if shape.members: + writer.line() + for member in shape.members.values(): + value = member.get_trait("enumValue") + writer.line( + self.symbols.member_name(member.id), + " = ", + _literal(member.name if value is None else value), + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if not docs and not shape.members: + writer.line("pass") + + +def _model_exports(plan: ModelPlan) -> tuple[str, ...]: + exports = ("ServiceError", *plan.registry_index) + return (*exports, "Unit") if plan.includes_unit else exports + + +def _write_mapping(writer: PythonWriter, name: str, values: Mapping[str, str]) -> None: + if not values: + writer.line(name, " = {}") + return + writer.line(name, " = {") + with writer.indent(): + for key, value in values.items(): + writer.line(_quote(key), ": ", _quote(value), ",") + writer.line("}") + + +def _write_string_set( + writer: PythonWriter, + name: str, + values: Collection[str], + *, + suffix: str = "", +) -> None: + if not values: + writer.line(name, "=frozenset()", suffix) + return + writer.line(name, "=frozenset(") + with writer.indent(): + writer.line("{") + with writer.indent(): + for value in sorted(values): + writer.line(_quote(value), ",") + writer.line("}") + writer.line(")", suffix) + + +def _write_docstring(writer: PythonWriter, value: str) -> None: + lines = [ + wrapped + for line in value.split("\n") + for wrapped in ( + wrap( + line, + width=_DOCSTRING_WIDTH, + break_long_words=False, + break_on_hyphens=False, + ) + or [""] + ) + ] + if len(lines) > 1: + writer.line('"""') + for line in lines: + writer.line(line.replace("\\", "\\\\").replace('"""', '\\"""')) + writer.line('"""') + return + writer.line(_quote(lines[0])) + + +def _literal(value: object) -> str: + return _quote(value) if isinstance(value, str) else repr(value) + + +def _quote(value: str) -> str: + return dumps(value, ensure_ascii=False) + + +def _join_types( + types: Collection[_symbols.TypeReference], +) -> tuple[str | _symbols.TypeReference, ...]: + parts: list[str | _symbols.TypeReference] = [] + for reference in types: + if parts: + parts.append(" | ") + parts.append(reference) + return tuple(parts) + + +def _documentation(shape: _model.Shape) -> str | None: + value = shape.get_trait("documentation") + return value if isinstance(value, str) else None + + +def _validate_member_names(context: GenerationContext[TypesSettings]) -> None: + reserved = _model_reserved_member_names(context.model) + for shape in context.plan.models.shapes: + names: set[str] = set() + shape_reserved = reserved(shape) + for member in shape.members.values(): + name = context.symbols.member_name(member.id) + if name in shape_reserved: + raise _exceptions.CodegenError( + f"Symbol provider did not reserve model member name {name!r} " + f"for {member.id}" + ) + if name in names: + raise _exceptions.CodegenError( + f"Generated model member name collision {name!r} in {shape.id}" + ) + names.add(name) + + +def _model_reserved_member_names( + model: _model.Model, +) -> Callable[[_model.Shape], Collection[str]]: + outputs = { + operation.output for operation in model.iter_shapes(_model.OperationShape) + } | {shape.id for shape in model.iter_shapes() if shape.has_trait("output")} + + def reserved(shape: _model.Shape) -> Collection[str]: + names = set(_MODEL_MEMBER_NAMES) + if shape.has_trait("error"): + names.update(_ERROR_MEMBER_NAMES) + if shape.has_trait("error") or shape.id in outputs: + names.add("response_metadata") + return names + + return reserved diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 1fe1d417e..3dfa2be3b 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -69,6 +69,13 @@ def __init__( self._union_variant_names: dict[ShapeId, str] = {} self._union_unknown_names: dict[ShapeId, str] = {} self._selected = frozenset(shape.id for shape in selection.shapes) + self._reserved_declarations = _RESERVED_DECLARATIONS + if any( + shape.type not in _ENUMS + and any(member.target == _UNIT for member in shape.members.values()) + for shape in selection.shapes + ): + self._reserved_declarations |= {"Unit"} self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: @@ -84,7 +91,7 @@ def __init__( members: dict[str, ShapeId] = {} for member in shape.members.values(): self._check_collision(members, self.member_name(member.id), member.id) - self._shape_names = frozenset(declarations) + self._shape_names = frozenset((*declarations, *self._reserved_declarations)) for shape in selection.shapes: if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): continue @@ -106,7 +113,7 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: raise ModelError(f"No generated declaration for {shape.id}") service = self._selection.service name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name - result = _name(name, pascal=True) + result = _name(name, pascal=True, reserved=self._reserved_declarations) if not re.fullmatch(r"[A-Za-z0-9_]+", name) or not result.isidentifier(): raise ModelError( f"Invalid Python declaration name {result!r} for {shape.id}" @@ -211,7 +218,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: pending.append((current, True)) pending.extend((target, False) for target in reversed(targets)) elif current == ShapeId("smithy.api", "Unit"): - resolved[current] = TypeReference("None") + resolved[current] = TypeReference("Unit", f"{self._package}.models") elif shape.type in (ShapeType.STRUCTURE, ShapeType.UNION): resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" @@ -237,6 +244,7 @@ def member_name(self, shape_id: ShapeId | str) -> str: return _name( member.name, constant=container.type in _ENUMS, + preserve_underscores=True, reserved=self._reserved_member_names.get(container.id, ()), ) @@ -279,6 +287,8 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: "not or pass raise return try while with yield".split() ) _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_RESERVED_DECLARATIONS = frozenset({"ServiceError"}) +_UNIT = ShapeId("smithy.api", "Unit") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") @@ -321,6 +331,7 @@ def _name( *, pascal: bool = False, constant: bool = False, + preserve_underscores: bool = False, reserved: Collection[str] = (), ) -> str: if pascal: @@ -328,16 +339,34 @@ def _name( else: value = _ACRONYM_BOUNDARY.sub(r"\1_\2", value) value = _WORD_BOUNDARY.sub(r"\1_\2", value) - result = "_".join(word.lower() for word in value.split("_") if word) - if constant: - result = result.upper() + result = ( + value.lower() + if preserve_underscores + else "_".join(word.lower() for word in value.split("_") if word) + ) if result[:1].isdigit(): result = "_" + result - if result in _KEYWORDS or result in reserved: + if result.startswith("__") and not result.endswith("__"): + result += "___" + elif ( + result in _KEYWORDS + or result in reserved + or (constant and result.upper() in reserved) + ): result += "_" + if constant: + result = _escape_enum_underscores(result).upper() return result +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" + return name + + def _plan_schema_names(selection: Selection) -> dict[ShapeId, str]: names: dict[ShapeId, str] = {} used: set[str] = set() diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index d91ffdf2e..efcf45e4b 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -58,11 +58,16 @@ class PythonWriter: """Write trusted Python lines, retaining type references until rendering.""" def __init__( - self, module: str, *, dependencies: _dependencies.DependencySet | None = None + self, + module: str, + *, + dependencies: _dependencies.DependencySet | None = None, + stub: bool = False, ) -> None: _validate_module(module, relative=False) self._module = _binding(module) self._dependencies = dependencies + self._stub = stub self._depth = 0 self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] @@ -91,9 +96,10 @@ def render(self) -> str: names, imports = self._plan_imports(*self._names()) header = "from __future__ import annotations\n" if imports: - header += "\n" + "\n".join(item.statement() for item in imports) + "\n" + header += "\n" + _render_imports(imports) + "\n" body = self._body(lambda reference: self._annotation(reference, names)) - return header + ("\n\n" + "\n".join(body) + "\n" if body else "") + separator = "\n" if self._stub else "\n\n" + return header + (separator + "\n".join(body) + "\n" if body else "") def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -139,6 +145,7 @@ def _plan_imports( self, module_names: frozenset[str], shadows: list[frozenset[str]] ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: identities: set[tuple[str | None, str]] = set() + third_party_roots: set[str] = set() # The local names visible where each identity is referenced. References # in one scope share a set, so keep the distinct sets, not their union. local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} @@ -153,6 +160,8 @@ def _plan_imports( if ref.module is not None: _validate_identifier(ref.name, "reference name") _validate_module(ref.module, relative=True) + if ref.dependencies and not ref.module.startswith("."): + third_party_roots.add(_module_root(ref.module)) identities.add((ref.module, ref.name)) key = _identity(ref.module, ref.name) local_names.setdefault(key, set()).add(shadow) @@ -195,7 +204,17 @@ def _plan_imports( else name ) names[(module, name)] = alias - imports.append((_Import(module, name, alias), scoped)) + imports.append( + ( + _Import( + module, + name, + alias, + third_party=_module_root(module) in third_party_roots, + ), + scoped, + ) + ) qualified: set[frozenset[str]] = set() for key, (module, name) in representatives.items(): module_binding, name_binding = key @@ -211,7 +230,9 @@ def _plan_imports( elif module_binding == self._module: names[(module, name)] = name if qualified: - imports.append((_Import("builtins", None, "_builtins"), qualified)) + imports.append( + (_Import("builtins", None, "_builtins", third_party=False), qualified) + ) # Check the complete plan, including unaliased imports. No binding gets # priority merely because its reference was encountered first. @@ -273,12 +294,7 @@ class _Import: module: str name: str | None alias: str - - def statement(self) -> str: - if self.name is None: - return f"import {self.module} as {self.alias}" - suffix = f" as {self.alias}" if self.alias != self.name else "" - return f"from {self.module} import {self.name}{suffix}" + third_party: bool @dataclass(eq=False, slots=True) @@ -319,11 +335,50 @@ def _binding(name: str) -> str: return normalize("NFKC", name) +def _render_imports(imports: list[_Import]) -> str: + sections: dict[int, dict[str, list[_Import]]] = {} + for item in imports: + section = 2 if item.module.startswith(".") else 1 if item.third_party else 0 + sections.setdefault(section, {}).setdefault(item.module, []).append(item) + + rendered: list[str] = [] + for section in sorted(sections): + lines: list[str] = [] + for module, items in sections[section].items(): + for item in items: + if item.name is None: + lines.append(f"import {module} as {item.alias}") + named = [item for item in items if item.name is not None] + if named: + lines.extend(_render_from_import(module, named)) + rendered.append("\n".join(lines)) + return "\n\n".join(rendered) + + +def _render_from_import(module: str, imports: list[_Import]) -> list[str]: + names: list[str] = [] + for item in imports: + if item.name is None: + continue + names.append( + item.name if item.alias == item.name else f"{item.name} as {item.alias}" + ) + prefix = f"from {module} import " + joined = ", ".join(names) + if len(prefix) + len(joined) <= 88: + return [prefix + joined] + return [prefix + "(", *(f" {name}," for name in names), ")"] + + def _identity(module: str | None, name: str) -> tuple[str | None, str]: """Return a reference's module and name as Python binds them.""" return (None if module is None else _binding(module), _binding(name)) +def _module_root(module: str) -> str: + return _binding(module.lstrip(".").split(".", 1)[0]) + + def _validate_identifier(value: str, kind: str) -> None: """Require a Python identifier that is not a hard keyword.""" if not value.isidentifier() or keyword.iskeyword(_binding(value)): diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py new file mode 100644 index 000000000..1192f205d --- /dev/null +++ b/packages/smithy-python/tests/unit/test_models.py @@ -0,0 +1,878 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import importlib +import json +import pickle +import subprocess +import sys +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest +from smithy_core.types import UnknownEnumMixin +from smithy_json import JSONCodec +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names +from smithy_python.schemas import SchemaGenerator +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + +_PACKAGE = "generated_types" + + +def _run_tool(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _context( + model_data: Mapping[str, object], + *, + service_id: str | None = "example#Service", + package: str = _PACKAGE, +) -> GenerationContext[TypesSettings]: + model = load_model(model_data) + selection = select_type_shapes( + model, + service_id=service_id, + generate_inputs_and_outputs=True, + ) + settings = TypesSettings(package, "1.0.0", None) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) + dependencies = DependencySet() + return GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + + +def _model() -> dict[str, object]: + return { + "smithy": "2.0", + "shapes": { + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "rename": { + "example#Original": "Renamed", + "example#Choice": "Selection", + "example#Color": "Tone", + }, + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + "errors": [{"target": "example#Problem"}], + }, + "example#Input": { + "type": "structure", + "traits": {"smithy.api#documentation": "Input docs."}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + "list": {"target": "smithy.api#String"}, + "renamed": {"target": "example#Original"}, + "choice": {"target": "example#Choice"}, + "color": {"target": "example#Color"}, + "stream": {"target": "example#Stream"}, + }, + }, + "example#Output": { + "type": "structure", + "members": { + "responseMetadata": {"target": "smithy.api#String"}, + "code": {"target": "example#Code"}, + }, + }, + "example#Problem": { + "type": "structure", + "traits": { + "smithy.api#error": "client", + "smithy.api#documentation": "Problem docs.", + }, + "members": { + "message": { + "target": "smithy.api#String", + "traits": {"smithy.api#sensitive": {}}, + }, + "fault": {"target": "smithy.api#String"}, + "detail": {"target": "example#Original"}, + }, + }, + "example#Original": { + "type": "structure", + "traits": {"smithy.api#documentation": "Renamed structure."}, + "members": { + "value": { + "target": "smithy.api#String", + "traits": { + "smithy.api#required": {}, + "smithy.api#documentation": "Value docs.", + }, + }, + "count": { + "target": "smithy.api#PrimitiveInteger", + "traits": {"smithy.api#required": {}}, + }, + }, + }, + "example#Choice": { + "type": "union", + "traits": {"smithy.api#documentation": "Choice docs."}, + "members": { + "text": { + "target": "smithy.api#String", + "traits": {"smithy.api#documentation": "Text docs."}, + }, + "item": {"target": "example#Original"}, + }, + }, + "example#Color": { + "type": "enum", + "traits": {"smithy.api#documentation": "Color docs."}, + "members": { + "RED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "red", + "smithy.api#documentation": "Red docs.", + }, + } + }, + }, + "example#Code": { + "type": "intEnum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 200}, + } + }, + }, + "example#Stream": { + "type": "blob", + "traits": {"smithy.api#streaming": {}}, + }, + }, + } + + +def _generate(tmp_path: Path) -> tuple[GenerationContext[TypesSettings], Path]: + context = _context(_model()) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{_PACKAGE}/__init__.py", "") + context.files.text_file(f"src/{_PACKAGE}/_private/__init__.py", "") + context.files.commit(tmp_path) + return context, tmp_path / "src" + + +def _generate_custom( + tmp_path: Path, + model: Mapping[str, object], + package: str, +) -> Path: + context = _context(model, service_id=None, package=package) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{package}/__init__.py", "") + context.files.text_file(f"src/{package}/_private/__init__.py", "") + context.files.commit(tmp_path) + return tmp_path / "src" + + +def test_model_plan_precomputes_registry_metadata() -> None: + plan = _context(_model()).plan.models + + assert plan.renames == { + "Renamed": "RENAMED", + "Selection": "SELECTION", + "Tone": "TONE", + } + assert "Original" not in plan.registry_index + assert plan.registry_index["SelectionText"] == "SELECTION" + assert plan.registry_index["SelectionUnknown"] == "SELECTION" + assert plan.target_names["example#Original"] == "Renamed" + assert plan.target_names["example#Choice"] == "Selection" + assert plan.target_names["example#Color"] == "Tone" + assert tuple(plan.registry_index) == ( + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "SelectionText", + "SelectionItem", + "SelectionUnknown", + "Tone", + "Code", + ) + assert plan.shape_names == { + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "Tone", + "Code", + } + + +def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: + context, source_root = _generate(tmp_path) + + runtime = (source_root / _PACKAGE / "models.py").read_text() + stub = (source_root / _PACKAGE / "models.pyi").read_text() + assert "from smithy_core._dynamic_models import ModelRegistry" in runtime + service_error_doc = ( + "class ServiceError(ModeledError):\n" + ' """\n' + " Base error for all errors in the service.\n" + "\n" + " Some exceptions do not extend from this class, including synthetic,\n" + " implicit, and shared exception types.\n" + ' """' + ) + assert service_error_doc in runtime + assert '"Renamed": "RENAMED"' in runtime + assert '"Original": "RENAMED"' not in runtime + assert "_MODEL_NAMES = tuple(_MODEL_INDEX)" in runtime + assert '__all__ = ("ServiceError", *_MODEL_NAMES)' in runtime + assert "globals()[name] = value" in runtime + assert "def __dir__() -> list[str]:" in runtime + + assert "class Input(_SerdeShape):" in stub + assert "from smithy_core.aio.interfaces import StreamingBlob" in stub + assert service_error_doc in stub + assert '"Input docs."' in stub + assert "required: str | None = ..." in stub + assert "list_: str | None = ..." in stub + assert "stream: StreamingBlob | None = ..." in stub + assert "class Problem(ServiceError, _SerdeShape):" in stub + assert "message:" not in stub + assert "fault_: str | None = ..." in stub + assert "class Renamed(_SerdeShape):" in stub + assert '"Value docs."' in stub + assert "count: int | None = ..." in stub + assert "class SelectionText(_SerdeShape):" in stub + assert "Selection = SelectionText | SelectionItem | SelectionUnknown" in stub + assert "class Tone(UnknownEnumMixin, StrEnum):" in stub + assert 'RED = "red"' in stub + assert "class Code(UnknownEnumMixin, IntEnum):" in stub + assert "OK = 200" in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + assert SMITHY_CORE in context.dependencies + + +def test_generated_models_are_lazy_and_runtime_compatible( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate(tmp_path) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + + assert "Renamed" not in vars(models) + assert "Renamed" in dir(models) + renamed = models.Renamed + assert vars(models)["Renamed"] is renamed + assert models.Renamed is renamed + assert renamed.__module__ == f"{_PACKAGE}.models" + value = renamed(value="hello") + assert value.value == "hello" + assert value.count == 0 + assert pickle.loads(pickle.dumps(value)) == value + + def get_input(_: int) -> type[Any]: + return models.Input + + with ThreadPoolExecutor(max_workers=8) as executor: + inputs = tuple(executor.map(get_input, range(32))) + assert len({id(model) for model in inputs}) == 1 + input_ = inputs[0]( + required="yes", + list_="value", + renamed=value, + choice=models.SelectionText(value="selected"), + color=models.Tone.RED, + ) + assert input_.list_ == "value" + assert input_.renamed is value + + stream = b"stream" + stream_input = models.Input(stream=stream) + assert stream_input.stream is stream + + class StreamSerializer: + value: object | None = None + + def write_data_stream(self, schema: Any, value: object) -> None: + self.value = value + + serializer = StreamSerializer() + stream_input.serialize_members(serializer) + assert serializer.value is stream + + class StreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + member = next( + member + for member in schema.members_by_index + if member.expect_member_name() == "stream" + ) + consumer(member, self) + + def read_data_stream(self, schema: Any) -> bytes: + return stream + + decoded_stream = models.Input.deserialize(StreamDeserializer()) + assert decoded_stream.stream is stream + + decoded = JSONCodec().deserialize( + ( + b'{"required":"yes","renamed":{"value":"hello"},' + b'"choice":{"text":"selected"},"color":"red"}' + ), + inputs[0], + ) + assert type(decoded.renamed) is renamed + assert decoded.renamed.value == "hello" + assert type(decoded.choice) is models.SelectionText + assert decoded.choice.value == "selected" + assert decoded.color is models.Tone.RED + assert json.loads(JSONCodec().serialize(input_)) == { + "required": "yes", + "list": "value", + "renamed": {"value": "hello", "count": 0}, + "choice": {"text": "selected"}, + "color": "red", + } + + assert issubclass(models.Problem, models.ServiceError) + problem = models.Problem("bad", fault_="client", detail=value) + assert problem.message == "bad" + assert problem.fault_ == "client" + assert "bad" not in repr(problem) + assert models.Problem().message == "" + + assert models.Tone.RED == "red" + assert issubclass(models.Tone, UnknownEnumMixin) + assert models.Code.OK == 200 + assert models.SelectionText(value="text").value == "text" + assert models.SelectionItem(value=value).value is value + assert models.SelectionUnknown(tag="future").tag == "future" + + namespace: dict[str, object] = {} + exec(f"from {_PACKAGE}.models import *", namespace) + assert tuple(name for name in models.__all__ if name not in namespace) == () + assert "Original" not in models.__all__ + with pytest.raises(AttributeError): + models.Original + + +def test_generated_model_names_do_not_shadow_runtime_helpers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_collisions" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + f"example#{name}": {"type": "structure"} + for name in ("ServiceError", "ModelRegistry", "ModeledError") + }, + }, + package, + ) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + + assert models.ServiceError_.__name__ == "ServiceError_" + assert models.ModelRegistry.__name__ == "ModelRegistry" + assert models.ModeledError.__name__ == "ModeledError" + assert models.ServiceError is not models.ServiceError_ + assert {"ServiceError_", "ModelRegistry", "ModeledError"} <= set(models.__all__) + + +def test_explicit_input_and_output_traits_match_runtime_and_stub( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_io_traits" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + } + }, + }, + "example#Output": { + "type": "structure", + "traits": {"smithy.api#output": {}}, + "members": {"responseMetadata": {"target": "smithy.api#String"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "required: str | None = ..." in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Input().required is None + output = models.Output(response_metadata_="modeled") + assert output.response_metadata_ == "modeled" + assert output.response_metadata is not None + + +def test_stub_and_runtime_use_the_same_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_member_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Leading": { + "type": "structure", + "members": {"_value": {"target": "smithy.api#String"}}, + }, + "example#Trailing": { + "type": "structure", + "members": {"value_": {"target": "smithy.api#String"}}, + }, + "example#Repeated": { + "type": "structure", + "members": {"some__value": {"target": "smithy.api#String"}}, + }, + "example#Digits": { + "type": "structure", + "members": { + "_1": {"target": "smithy.api#String"}, + "_2HTTPServer": {"target": "smithy.api#String"}, + }, + }, + "example#RuntimeMethods": { + "type": "structure", + "members": { + "serialize": {"target": "smithy.api#String"}, + "serializeMembers": {"target": "smithy.api#String"}, + "deserialize": {"target": "smithy.api#String"}, + "__init__": {"target": "smithy.api#String"}, + "__post_init__": {"target": "smithy.api#String"}, + "_smithy_default": {"target": "smithy.api#String"}, + "__repr__": {"target": "smithy.api#String"}, + }, + }, + "example#Keywords": { + "type": "enum", + "members": { + name: { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": name.lower()}, + } + for name in ("LIST", "CLASS") + }, + }, + "example#SunderString": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "value"}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "dunder"}, + }, + }, + }, + "example#SunderInt": { + "type": "intEnum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 1}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 2}, + }, + }, + }, + "example#EnumContainer": { + "type": "structure", + "members": { + "stringValue": {"target": "example#SunderString"}, + "intValue": {"target": "example#SunderInt"}, + }, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "_value: str | None = ..." in stub + assert "value_: str | None = ..." in stub + assert "some__value: str | None = ..." in stub + assert "_1: str | None = ..." in stub + assert "_2_http_server: str | None = ..." in stub + assert "serialize_: str | None = ..." in stub + assert "serialize_members_: str | None = ..." in stub + assert "deserialize_: str | None = ..." in stub + assert "__init___: str | None = ..." in stub + assert "__post_init___: str | None = ..." in stub + assert "_smithy_default_: str | None = ..." in stub + assert "__repr___: str | None = ..." in stub + assert 'LIST_ = "list"' in stub + assert 'CLASS_ = "class"' in stub + assert ( + 'class SunderString(UnknownEnumMixin, StrEnum):\n _VALUE__ = "value"' in stub + ) + assert "class SunderInt(UnknownEnumMixin, IntEnum):\n _VALUE__ = 1" in stub + assert '__DUNDER___ = "dunder"' in stub + assert "__DUNDER___ = 2" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Leading(_value="leading")._value == "leading" + assert models.Trailing(value_="trailing").value_ == "trailing" + assert models.Repeated(some__value="repeated").some__value == "repeated" + digits = models.Digits(_1="one", _2_http_server="two") + assert digits._1 == "one" + assert digits._2_http_server == "two" + methods = models.RuntimeMethods( + serialize_="serialize", + serialize_members_="members", + deserialize_="deserialize", + __init___="init", + __post_init___="post_init", + _smithy_default_="default", + __repr___="repr", + ) + assert callable(methods.serialize) + assert callable(methods.serialize_members) + assert callable(methods.deserialize) + assert methods.__init___ == "init" + assert methods.__post_init___ == "post_init" + assert methods._smithy_default_ == "default" + assert methods.__repr___ == "repr" + assert isinstance(methods._smithy_default(), models.RuntimeMethods) + assert "RuntimeMethods(" in repr(methods) + assert json.loads(JSONCodec().serialize(methods)) == { + "serialize": "serialize", + "serializeMembers": "members", + "deserialize": "deserialize", + "__init__": "init", + "__post_init__": "post_init", + "_smithy_default": "default", + "__repr__": "repr", + } + assert models.Keywords.LIST_ == "list" + assert models.Keywords.CLASS_ == "class" + assert models.SunderString._VALUE__ == "value" + assert models.SunderInt._VALUE__ == 1 + assert "__DUNDER___" in models.SunderString.__members__ + assert "__DUNDER___" in models.SunderInt.__members__ + value = models.EnumContainer( + string_value=models.SunderString.__DUNDER___, + int_value=models.SunderInt.__DUNDER___, + ) + encoded = JSONCodec().serialize(value) + decoded = JSONCodec().deserialize(encoded, models.EnumContainer) + assert decoded.string_value is models.SunderString.__DUNDER___ + assert decoded.int_value is models.SunderInt.__DUNDER___ + + +def test_unit_union_members_round_trip_as_empty_structures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_union" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + runtime = (source_root / package / "models.py").read_text() + stub = (source_root / package / "models.pyi").read_text() + assert '_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit")' in runtime + assert "class ActionQuit(_SerdeShape):\n value: Unit" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + value = models.Container(action=models.ActionQuit(value=models.Unit())) + assert json.loads(JSONCodec().serialize(value)) == {"action": {"quit": {}}} + decoded = JSONCodec().deserialize(b'{"action":{"quit":{}}}', models.Container) + assert isinstance(decoded.action, models.ActionQuit) + assert isinstance(decoded.action.value, models.Unit) + assert "Unit" in models.__all__ + + +def test_modeled_unit_does_not_shadow_prelude_unit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_collision" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Unit": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "class Unit_(_SerdeShape):" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Unit_ is not models.Unit + decoded = JSONCodec().deserialize( + b'{"action":{"quit":{}}}', + models.Container, + ) + assert isinstance(decoded.action.value, models.Unit) + + +def test_generated_stub_passes_strict_pyright( + tmp_path: Path, +) -> None: + _, source_root = _generate(tmp_path) + (tmp_path / "check.py").write_text( + ( + f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" + "renamed = Renamed(value='value')\n" + "Input(required='required', renamed=renamed, " + "choice=SelectionText(value='x'), color=Tone.RED, stream=b'payload')\n" + "problem = Problem('bad')\n" + "message: str = problem.message\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_models_pass_ruff(tmp_path: Path) -> None: + _, source_root = _generate(tmp_path) + models = source_root / _PACKAGE / "models.py" + stub = source_root / _PACKAGE / "models.pyi" + for command in (("check",), ("format", "--check")): + result = _run_tool( + "ruff", + *command, + str(models), + str(stub), + "--config", + str(Path(__file__).resolve().parents[4] / "pyproject.toml"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stub_star_exports_and_private_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_private_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#_2HTTPServer": { + "type": "structure", + "members": { + "__value": {"target": "smithy.api#String"}, + "__annotations__": {"target": "smithy.api#String"}, + }, + } + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert '"_2HTTPServer",' in stub + assert "__value___: str | None = ..." in stub + assert "__annotations___: str | None = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + namespace: dict[str, object] = {} + exec(f"from {package}.models import *", namespace) + assert namespace["_2HTTPServer"] is models._2HTTPServer + value = models._2HTTPServer( + __value___="value", + __annotations___="annotations", + ) + assert value.__value___ == "value" + assert value.__annotations___ == "annotations" + + (tmp_path / "check.py").write_text( + ( + f"from {package}.models import *\n\n" + "value = _2HTTPServer(\n" + " __value___='value',\n" + " __annotations___='annotations',\n" + ")\n" + "member: str | None = value.__value___\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_model_output_is_deterministic(tmp_path: Path) -> None: + outputs: list[tuple[str, str]] = [] + for index in range(2): + _, source_root = _generate(tmp_path / str(index)) + package = source_root / _PACKAGE + outputs.append( + ( + (package / "models.py").read_text(), + (package / "models.pyi").read_text(), + ) + ) + assert outputs[0] == outputs[1] + + +def test_model_generation_without_service(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Standalone": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + } + }, + }, + service_id=None, + ) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + runtime = (tmp_path / "src" / _PACKAGE / "models.py").read_text() + stub = (tmp_path / "src" / _PACKAGE / "models.pyi").read_text() + assert '"Standalone": "STANDALONE"' in runtime + assert "class Standalone(_SerdeShape):" in stub + + +def teardown_module() -> None: + for name in tuple(sys.modules): + if name == _PACKAGE or name.startswith(f"{_PACKAGE}."): + del sys.modules[name] diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 4e0c77866..a1013999f 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -25,10 +25,13 @@ def model_with(shapes: dict[str, object]) -> Model: ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), ("HTTP2Server", "HTTP2Server", "http2_server", "HTTP2_SERVER"), ("getURL2Value", "GetURL2Value", "get_url2_value", "GET_URL2_VALUE"), - ("__some__name__", "SomeName", "some_name", "SOME_NAME"), - ("__init__", "Init", "init", "INIT"), - ("_value", "Value", "value", "VALUE"), - ("class", "Class", "class_", "CLASS"), + ("__some__name__", "SomeName", "__some__name__", "__SOME__NAME___"), + ("__init__", "Init", "__init__", "__INIT___"), + ("__post_init__", "PostInit", "__post_init__", "__POST_INIT___"), + ("__value", "Value", "__value___", "__VALUE___"), + ("_value", "Value", "_value", "_VALUE"), + ("_VALUE_", "VALUE", "_value_", "_VALUE__"), + ("class", "Class", "class_", "CLASS_"), ("None", "None_", "none", "NONE"), ("match", "Match", "match", "MATCH"), ], @@ -59,6 +62,21 @@ def test_names(source: str, declaration: str, field: str, constant: str) -> None assert model.get_shape(f"example#Choice${source}").get_trait("enumValue") == "wire" +def test_service_error_is_reserved_for_the_synthetic_error_base() -> None: + model = model_with( + { + "example#ServiceError": {"type": "structure"}, + "example#Service": { + "type": "union", + "members": {"error": {"target": "smithy.api#String"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + assert symbols.declaration_name("example#ServiceError") == "ServiceError_" + assert symbols.union_variant_name("example#Service$error") == "Service_Error" + + @pytest.mark.parametrize( ("kind", "name", "module"), [ @@ -99,7 +117,9 @@ def test_type_references(kind: str, name: str, module: str) -> None: if module != "example.client.models" and kind not in ("enum", "intEnum"): prelude_name = kind[0].upper() + kind[1:] assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref - assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") + assert symbols.type_reference("smithy.api#Unit") == TypeReference( + "Unit", "example.client.models" + ) def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: @@ -403,7 +423,8 @@ def test_collection_only_cycles(mutual: bool) -> None: @pytest.mark.parametrize("kind", ["structure", "union", "enum", "intEnum"]) @pytest.mark.parametrize( - "names", [("getURL", "get_url"), ("class", "class_"), ("__init__", "init")] + "names", + [("getURL", "get_url"), ("class", "class_"), ("HTTPServer", "http_server")], ) def test_member_collisions(kind: str, names: tuple[str, str]) -> None: from smithy_python.model import ModelError @@ -420,7 +441,7 @@ def test_member_collisions(kind: str, names: tuple[str, str]) -> None: SymbolProvider(model, select_shapes(model), package="pkg") for name in names: assert f"example#Container${name}" in str(error.value) - result = "CLASS" if kind in ("enum", "intEnum") else "class_" + result = "CLASS_" if kind in ("enum", "intEnum") else "class_" if names[0] == "class": assert repr(result) in str(error.value) @@ -786,6 +807,54 @@ def test_reserved_member_names_still_collide() -> None: assert "'serialize_'" in str(error.value) +def test_enum_sunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "_VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="_VALUE__"): + SymbolProvider(model, select_shapes(model), package="pkg") + + +def test_enum_dunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "__VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "__VALUE___": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="__VALUE___"): + SymbolProvider(model, select_shapes(model), package="pkg") + + def test_type_references_are_cached() -> None: from smithy_python.model import ModelError diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index 78852e0cc..c7df26470 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -13,7 +13,7 @@ from typing import get_type_hints import pytest -from smithy_python.dependencies import Dependency, DependencySet +from smithy_python.dependencies import SMITHY_CORE, Dependency, DependencySet from smithy_python.exceptions import CodegenError from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter @@ -220,7 +220,7 @@ def test_class_body_references_load( ")", ) source = writer.render() - assert "from dataclasses import field as _dataclasses_field" in source + assert "field as _dataclasses_field" in source assert "default_factory=_builtins.list" in source module = load_written_module(writer, tmp_path, monkeypatch) node: type = module.Node @@ -600,7 +600,7 @@ def test_builtin_names_cover_running_interpreter() -> None: writer.line("value: ", TypeReference(name, "external")) source = writer.render() for name in names: - assert f"from external import {name} as _external_{name}\n" in source + assert f" {name} as _external_{name},\n" in source @pytest.mark.parametrize("module", ["written_models", "\uff57ritten_models"]) @@ -789,3 +789,28 @@ def test_import_identity_and_order() -> None: assert prefix.index("from datetime") < prefix.index("from decimal") prefixes.append(prefix) assert prefixes[0] == prefixes[1] + + +def test_imports_are_grouped_and_combined() -> None: + writer = PythonWriter("package.models") + writer.line("first: ", TypeReference("StrEnum", "enum")) + writer.line("second: ", TypeReference("IntEnum", "enum")) + writer.line( + "third: ", + TypeReference( + "Document", + "smithy_core.documents", + dependencies=(SMITHY_CORE,), + ), + ) + writer.line("fourth: ", TypeReference("Thing", "._private")) + + assert writer.render().startswith( + "from __future__ import annotations\n" + "\n" + "from enum import IntEnum, StrEnum\n" + "\n" + "from smithy_core.documents import Document\n" + "\n" + "from ._private import Thing\n" + ) diff --git a/pyproject.toml b/pyproject.toml index 17b89a56f..59990158d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ lint = [ ] typing = [ "pyright>=1.1.400", + "typing-extensions>=4.14.0", ] changelog = [ "jinja2>=3.1.6" diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index d82b8bfe0..62a7edef9 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1541,6 +1541,9 @@ "load_model": "rex-function", "to_json": "rex-function" }, + "smithy_python.models": { + "ModelGenerator": "class" + }, "smithy_python.schemas": { "SchemaGenerator": "class", "SchemaPlan": "class" diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 54ea35f82..4306ffda4 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false """Regenerate the committed Python codegen snapshot.""" import hashlib @@ -12,6 +13,7 @@ from smithy_python.files import FileManifest from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names from smithy_python.schemas import SchemaGenerator from smithy_python.selection import select_type_shapes from smithy_python.symbols import SymbolProvider @@ -35,7 +37,12 @@ def main() -> None: package_version="1.0.0", package_description="Example Python codegen output", ) - symbols = SymbolProvider(model, selection, package=settings.package) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) dependencies = DependencySet() context = GenerationContext( model=model, @@ -47,6 +54,7 @@ def main() -> None: files=FileManifest(dependencies), ) SchemaGenerator().generate(context) + ModelGenerator().generate(context) context.files.commit(_OUTPUT) _format_output() @@ -57,7 +65,7 @@ def _format_output() -> None: args = [sys.executable, "-m", "ruff", command] if command == "check": args.append("--fix") - subprocess.run( + subprocess.run( # noqa: S603 [*args, str(_OUTPUT), "--config", str(config)], check=True, ) diff --git a/uv.lock b/uv.lock index 3ef01194f..d1d136491 100644 --- a/uv.lock +++ b/uv.lock @@ -815,6 +815,7 @@ test = [ ] typing = [ { name = "pyright" }, + { name = "typing-extensions" }, ] [package.metadata] @@ -828,7 +829,10 @@ test = [ { name = "pytest-asyncio", specifier = ">=0.25.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, ] -typing = [{ name = "pyright", specifier = ">=1.1.400" }] +typing = [ + { name = "pyright", specifier = ">=1.1.400" }, + { name = "typing-extensions", specifier = ">=4.14.0" }, +] [[package]] name = "smithy-test" @@ -846,11 +850,11 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "typing-extensions" -version = "4.13.2" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]]