From 6941029c33dcd23c5a8510a77d3b6a74cc8c8ecf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:54:33 -0400 Subject: [PATCH 1/5] Bump the gradle group in /codegen with 11 updates (#873) Bumps the gradle group in /codegen with 11 updates: | Package | From | To | | --- | --- | --- | | [gradle-wrapper](https://github.com/gradle/gradle) | `9.8.1` | `9.8.2` | | [software.amazon.smithy:smithy-model](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-codegen-core](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-aws-traits](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-aws-protocol-tests](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-protocol-tests](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-protocol-traits](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-protocol-test-traits](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-waiters](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-rules-engine](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | | [software.amazon.smithy:smithy-aws-endpoints](https://github.com/smithy-lang/smithy) | `1.74.0` | `1.74.1` | Updates `gradle-wrapper` from 9.8.1 to 9.8.2 - [Release notes](https://github.com/gradle/gradle/releases) - [Commits](https://github.com/gradle/gradle/compare/v9.8.1...v9.8.2) Updates `software.amazon.smithy:smithy-model` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-codegen-core` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-protocol-tests` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-tests` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-test-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-waiters` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-rules-engine` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-endpoints` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-codegen-core` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-protocol-tests` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-tests` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-protocol-test-traits` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-waiters` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-rules-engine` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) Updates `software.amazon.smithy:smithy-aws-endpoints` from 1.74.0 to 1.74.1 - [Release notes](https://github.com/smithy-lang/smithy/releases) - [Changelog](https://github.com/smithy-lang/smithy/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy/compare/1.74.0...1.74.1) --- updated-dependencies: - dependency-name: gradle-wrapper dependency-version: 9.8.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-model dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-codegen-core dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-protocol-tests dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-tests dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-test-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-waiters dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-rules-engine dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-endpoints dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-codegen-core dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-protocol-tests dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-tests dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-protocol-test-traits dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-waiters dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-rules-engine dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle - dependency-name: software.amazon.smithy:smithy-aws-endpoints dependency-version: 1.74.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: gradle ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From 8a280793e1d0a83bf333dc3bfafd831efa94507d Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:43:04 -0400 Subject: [PATCH 2/5] wire native types generation into the CLI --- .../types/.smithy-python-manifest.json | 11 +- codegen-output-py/types/README.md | 5 + codegen-output-py/types/pyproject.toml | 43 +++ .../types/src/example_types/__init__.py | 3 + .../src/example_types/_private/__init__.py | 1 + .../types/src/example_types/py.typed | 0 packages/smithy-python/README.md | 20 +- packages/smithy-python/pyproject.toml | 4 +- .../smithy-python/src/smithy_python/cli.py | 137 ++++++++-- .../src/smithy_python/symbols.py | 7 +- .../smithy-python/src/smithy_python/types.py | 134 ++++++++++ packages/smithy-python/tests/unit/test_cli.py | 246 ++++++++++++++++-- .../tests/unit/test_selection.py | 73 ++++-- .../smithy-python/tests/unit/test_symbols.py | 3 + scripts/api-snapshot/api.json | 3 + scripts/codegen-output-py/generate.py | 59 ++--- uv.lock | 6 + 17 files changed, 637 insertions(+), 118 deletions(-) create mode 100644 codegen-output-py/types/README.md create mode 100644 codegen-output-py/types/pyproject.toml create mode 100644 codegen-output-py/types/src/example_types/__init__.py create mode 100644 codegen-output-py/types/src/example_types/_private/__init__.py create mode 100644 codegen-output-py/types/src/example_types/py.typed create mode 100644 packages/smithy-python/src/smithy_python/types.py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index d0ae825ed..01fe60f9b 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,8 +1,13 @@ { "files": { - "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", - "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", - "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a" + "README.md": "03146affd4cdec7675387b787ff852400641bad4c0cfc5e80bb860d576acee4b", + "pyproject.toml": "8b49dfb34a75d368f1c264a8369fb4093af3b9618ec4a93173b28315578c808e", + "src/example_types/__init__.py": "6d153749d98df8adfe6264fb74d642917cbf2d6922b9211719dee35483dab943", + "src/example_types/_private/__init__.py": "5384bfdb2df380b6557cc7a71d16891415bccaa87699406e236f752c6415389f", + "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", + "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", + "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14", + "src/example_types/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "version": 1 } diff --git a/codegen-output-py/types/README.md b/codegen-output-py/types/README.md new file mode 100644 index 000000000..a448afa11 --- /dev/null +++ b/codegen-output-py/types/README.md @@ -0,0 +1,5 @@ + + +## example_types Types Package + +Example native Python codegen output diff --git a/codegen-output-py/types/pyproject.toml b/codegen-output-py/types/pyproject.toml new file mode 100644 index 000000000..ac59b5076 --- /dev/null +++ b/codegen-output-py/types/pyproject.toml @@ -0,0 +1,43 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +[project] +name = "example_types" +version = "1.0.0" +description = "Example native Python codegen output" +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", "example_types"] +license = {text = "Apache-2.0"} +dependencies = [ + "smithy-core~=0.10.0", +] + +[build-system] +requires = [ + "hatchling", +] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = [ + "src/example_types", +] + +[tool.pyright] +typeCheckingMode = "strict" +reportPrivateUsage = false +reportUnusedFunction = false +reportUnusedVariable = false +reportUnnecessaryComparison = false +reportUnusedClass = false +enableExperimentalFeatures = true +exclude = ["**/models.py"] + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true diff --git a/codegen-output-py/types/src/example_types/__init__.py b/codegen-output-py/types/src/example_types/__init__.py new file mode 100644 index 000000000..f81c022be --- /dev/null +++ b/codegen-output-py/types/src/example_types/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__version__: str = "1.0.0" diff --git a/codegen-output-py/types/src/example_types/_private/__init__.py b/codegen-output-py/types/src/example_types/_private/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/codegen-output-py/types/src/example_types/_private/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/codegen-output-py/types/src/example_types/py.typed b/codegen-output-py/types/src/example_types/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/smithy-python/README.md b/packages/smithy-python/README.md index 79db2d4bc..2392b8f02 100644 --- a/packages/smithy-python/README.md +++ b/packages/smithy-python/README.md @@ -1,19 +1,17 @@ # smithy-python -> [!WARNING] -> This package is an experimental scaffold. It does not generate code yet. The -> Java generator in the repository's `codegen` directory remains authoritative. - -`smithy-python` will provide Python-native code generation for Smithy models. -The initial command-line interface exposes the planned client and types generation -commands so that their top-level shape can be developed independently from the -generator implementation. +`smithy-python` provides Python-native code generation for Smithy models. +Standalone types generation is available; client generation remains under +development. ```console smithy-python generate client [OPTIONS] smithy-python generate types [OPTIONS] ``` -After validating their invocation options, both generation commands currently exit -with an error explaining that generation has not been implemented. The package is -included in workspace builds to validate its packaging and entry points. +`generate types` produces an installable package containing compact schemas, lazy +runtime models, and a complete static typing surface. Run +`smithy-python generate types --help` for artifact settings. + +`generate client` currently exits with an error explaining that client generation +has not been implemented. diff --git a/packages/smithy-python/pyproject.toml b/packages/smithy-python/pyproject.toml index cab33c833..2a3e2ec9e 100644 --- a/packages/smithy-python/pyproject.toml +++ b/packages/smithy-python/pyproject.toml @@ -26,7 +26,9 @@ classifiers = [ "Programming Language :: Python :: Free Threading :: 2 - Beta", "Topic :: Software Development :: Code Generators", ] -dependencies = [] +dependencies = [ + "packaging", +] [project.scripts] smithy-python = "smithy_python.cli:main" diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index a94d90e03..15f8012ab 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -11,7 +11,18 @@ from pathlib import Path from typing import BinaryIO, Final +from packaging.version import InvalidVersion, Version + from . import __version__ +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import files as _files +from . import generation as _generation +from . import model as _model +from . import models as _models +from . import selection as _selection +from . import symbols as _symbols +from . import types as _types from .environment import PluginEnvironment from .exceptions import CodegenError, InvalidInvocationError from .model import load_model @@ -26,10 +37,8 @@ @dataclass(frozen=True, slots=True) class _Invocation: - artifact: str model_source: bytes output_dir: Path - environment: PluginEnvironment def main( @@ -48,24 +57,19 @@ def main( try: invocation = _resolve_invocation(args, environ=environ, stdin=stdin) - except InvalidInvocationError as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 2 - except (CodegenError, OSError) as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 1 - - try: model = load_model(invocation.model_source) - selection = select_shapes( - model, - service_id=args.service, - require_service=invocation.artifact == "client", - ) + if args.artifact == "types": + selection = _generate_types(args, invocation.output_dir, model) + else: + selection = select_shapes( + model, + service_id=args.service, + require_service=True, + ) except InvalidInvocationError as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 2 - except CodegenError as error: + except (CodegenError, OSError) as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 1 @@ -74,6 +78,8 @@ def main( f"smithy-python: excluded {selection.excluded_count} eligible data shape(s) " f"outside the closure of service {selection.service.id}\n" ) + if args.artifact == "types": + return 0 sys.stderr.write(_GENERATION_NOT_IMPLEMENTED.format(artifact=args.artifact)) return 1 @@ -93,11 +99,36 @@ def _create_parser() -> argparse.ArgumentParser: generate = commands.add_parser("generate", help="Generate Python source") artifacts = generate.add_subparsers(dest="artifact", required=True) common = _common_artifact_options() - for name, help_text in ( - ("client", "Generate a client package"), - ("types", "Generate a standalone types package"), - ): - artifacts.add_parser(name, help=help_text, parents=[common]) + artifacts.add_parser("client", help="Generate a client package", parents=[common]) + types = artifacts.add_parser( + "types", help="Generate a standalone types package", parents=[common] + ) + types.add_argument( + "--package", + required=True, + help="Import package and distribution name for the generated artifact.", + ) + types.add_argument( + "--package-version", + required=True, + type=_package_version, + help="Version for the generated package.", + ) + types.add_argument( + "--package-description", + help='Package description; defaults to " types package".', + ) + types.add_argument( + "--selector", + type=_parse_selector, + default=_selection.Selector.identity(), + help="Smithy selector used to narrow generated declaration roots.", + ) + types.add_argument( + "--generate-inputs-and-outputs", + action="store_true", + help="Include operation input and output structures as generation roots.", + ) return parser @@ -175,8 +206,68 @@ def _resolve_invocation( raise InvalidInvocationError("Expected a Smithy JSON AST model") return _Invocation( - artifact=args.artifact, model_source=model_source, output_dir=output_dir, - environment=environment, ) + + +def _generate_types( + args: argparse.Namespace, + output_dir: Path, + model: _model.Model, +) -> _selection.Selection: + description = args.package_description or f"{args.package} types package" + settings = _generation.TypesSettings( + package=args.package, + package_version=args.package_version, + package_description=description, + selector=args.selector, + generate_inputs_and_outputs=args.generate_inputs_and_outputs, + ) + selection = _selection.select_type_shapes( + model, + service_id=args.service, + selector=settings.selector, + generate_inputs_and_outputs=settings.generate_inputs_and_outputs, + ) + try: + symbols = _symbols.SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_models._model_reserved_member_names( # pyright: ignore[reportPrivateUsage] + model + ), + ) + except ValueError as error: + raise InvalidInvocationError(str(error)) from error + + dependencies = _dependencies.DependencySet() + context = _generation.GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=_generation.GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=_files.FileManifest(dependencies), + ) + _types.TypesGenerator().generate(context) + context.files.commit(output_dir) + return selection + + +def _parse_selector(value: str) -> _selection.Selector: + try: + return _selection.Selector.parse(value) + except _exceptions.SelectorSyntaxError as error: + raise argparse.ArgumentTypeError(str(error)) from error + + +def _package_version(value: str) -> str: + if not value: + raise argparse.ArgumentTypeError("expected a non-empty value") + try: + return str(Version(value)) + except InvalidVersion as error: + raise argparse.ArgumentTypeError(str(error)) from error diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index dbc7df1e4..d4db8e837 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -54,11 +54,13 @@ def __init__( package: str, reserved_member_names: Callable[[Shape], Collection[str]] | None = None, ) -> None: - if any( + if not _DISTRIBUTION_NAME.fullmatch(package) or any( not part.isidentifier() or unicodedata.normalize("NFKC", part) in _KEYWORDS for part in package.split(".") ): - raise ValueError(f"Invalid Python package name: {package!r}") + raise ValueError( + f"Invalid Python package or distribution name: {package!r}" + ) self._model = model self._selection = selection self._package = package @@ -287,6 +289,7 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) _RESERVED_DECLARATIONS = frozenset({"ServiceError"}) _UNIT = ShapeId("smithy.api", "Unit") +_DISTRIBUTION_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") diff --git a/packages/smithy-python/src/smithy_python/types.py b/packages/smithy-python/src/smithy_python/types.py new file mode 100644 index 000000000..ef2b43560 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/types.py @@ -0,0 +1,134 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generation of complete standalone types packages.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + +from . import dependencies as _dependencies +from . import generation as _generation +from . import models as _models +from . import schemas as _schemas + +__all__ = ["TypesGenerator"] + + +class TypesGenerator: + """Generate a standalone package containing schemas and lazy models.""" + + def generate( + self, context: _generation.GenerationContext[_generation.TypesSettings] + ) -> None: + """Register every file and dependency in the generation context.""" + _schemas.SchemaGenerator().generate(context) + _models.ModelGenerator().generate(context) + _write_package(context) + + +_GENERATED_COMMENT = "# Code generated by smithy-python-codegen DO NOT EDIT." +_GENERATED_MARKDOWN_COMMENT = ( + "" +) +_PYPROJECT_SUFFIX = """ +[tool.pyright] +typeCheckingMode = "strict" +reportPrivateUsage = false +reportUnusedFunction = false +reportUnusedVariable = false +reportUnnecessaryComparison = false +reportUnusedClass = false +enableExperimentalFeatures = true +exclude = ["**/models.py"] + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true +""" + + +def _write_package( + context: _generation.GenerationContext[_generation.TypesSettings], +) -> None: + settings = context.settings + package = settings.package + directory = package.replace(".", "/") + description = settings.package_description or f"{package} types package" + + context.dependencies.add(_dependencies.HATCHLING) + context.files.text_file( + "pyproject.toml", + _render_pyproject(settings, context.dependencies, description), + ) + context.files.text_file( + "README.md", + f"{_GENERATED_MARKDOWN_COMMENT}\n\n" + f"## {package} Types Package\n\n{description}\n", + ) + + init = context.files.python_module(f"src/{directory}/__init__.py", package) + init.line("__version__: str = ", repr(settings.package_version)) + context.files.python_module( + f"src/{directory}/_private/__init__.py", f"{package}._private" + ) + context.files.text_file(f"src/{directory}/py.typed", "") + + +def _render_pyproject( + settings: _generation.TypesSettings, + dependencies: _dependencies.DependencySet, + description: str, +) -> str: + package_root = settings.package.split(".", 1)[0] + groups = "\n".join( + f"{group} = {_toml_array(map(_requirement, values))}" + for group in ("test", "docs") + if (values := dependencies.for_group(group)) + ) + if groups: + groups = f"\n\n[dependency-groups]\n{groups}" + + return f"""{_GENERATED_COMMENT} + +[project] +name = {_toml_string(settings.package)} +version = {_toml_string(settings.package_version)} +description = {_toml_string(description)} +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", {_toml_string(settings.package)}] +license = {{text = "Apache-2.0"}} +dependencies = {_toml_array(map(_requirement, dependencies.for_group("runtime")))}{groups} + +[build-system] +requires = {_toml_array(map(_requirement, dependencies.for_group("build")))} +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = {_toml_array((f"src/{package_root}",))} + +{_PYPROJECT_SUFFIX.lstrip()}""" + + +def _toml_array(values: Iterable[str]) -> str: + items = tuple(values) + return ( + "[\n" + "\n".join(f" {_toml_string(value)}," for value in items) + "\n]" + if items + else "[]" + ) + + +def _requirement(dependency: _dependencies.Dependency) -> str: + extras = f"[{','.join(dependency.extras)}]" if dependency.extras else "" + return f"{dependency.package}{extras}{dependency.specifier}" + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index 6f08e6178..a0d554608 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -3,8 +3,11 @@ from __future__ import annotations +import importlib +import json import subprocess import sys +import tomllib from io import BytesIO from pathlib import Path @@ -20,6 +23,27 @@ def isatty(self) -> bool: return True +def _types_args( + output: Path, + *options: str, + package: str = "example_types", + version: str = "1.2.3", +) -> tuple[str, ...]: + args = ("generate", "types", "--output", str(output)) + return (*args, "--package", package, "--package-version", version, *options) + + +def _run_types( + output: Path, + *options: str, + source: bytes = VALID_MODEL.encode(), + package: str = "example_types", + version: str = "1.2.3", +) -> int: + args = _types_args(output, *options, package=package, version=version) + return main(args, environ={}, stdin=BytesIO(source)) + + @pytest.mark.parametrize( ("argv", "expected"), [ @@ -34,9 +58,7 @@ def test_information_commands( assert capsys.readouterr().out.startswith(expected) -@pytest.mark.parametrize("artifact", ["client", "types"]) -def test_generation_commands_are_explicitly_unavailable( - artifact: str, +def test_client_generation_is_explicitly_unavailable( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -47,7 +69,7 @@ def test_generation_commands_are_explicitly_unavailable( main( ( "generate", - artifact, + "client", "--model", str(model), "--output", @@ -58,8 +80,147 @@ def test_generation_commands_are_explicitly_unavailable( == 1 ) assert capsys.readouterr().err == ( - f"smithy-python: error: {artifact} generation is not implemented yet\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + + +def test_types_generation_writes_complete_package( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = tmp_path / "model.json" + model.write_text( + '{"smithy": "2.0", "shapes": {"example#Data": {"type": "structure"}}}' ) + output = tmp_path / "output" + + assert _run_types(output, "--model", str(model)) == 0 + assert capsys.readouterr().err == "" + assert { + path.relative_to(output).as_posix() + for path in output.rglob("*") + if path.is_file() + } == { + ".smithy-python-manifest.json", + "README.md", + "pyproject.toml", + "src/example_types/__init__.py", + "src/example_types/_private/__init__.py", + "src/example_types/_private/schemas.py", + "src/example_types/models.py", + "src/example_types/models.pyi", + "src/example_types/py.typed", + } + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"] == { + "name": "example_types", + "version": "1.2.3", + "description": "example_types types package", + "readme": "README.md", + "requires-python": ">=3.12", + "keywords": ["smithy", "example_types"], + "license": {"text": "Apache-2.0"}, + "dependencies": ["smithy-core~=0.10.0"], + } + assert project["build-system"] == { + "requires": ["hatchling"], + "build-backend": "hatchling.build", + } + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("example_types") + models = importlib.import_module("example_types.models") + assert package.__version__ == "1.2.3" + assert models.Data.__module__ == "example_types.models" + + +def test_types_generation_escapes_del_in_toml_metadata(tmp_path: Path) -> None: + output = tmp_path / "output" + description = "before\x7fafter" + + assert _run_types(output, "--package-description", description) == 0 + pyproject = (output / "pyproject.toml").read_text() + assert "\\u007f" in pyproject + assert tomllib.loads(pyproject)["project"]["description"] == description + + +def test_types_generation_applies_selector_and_input_output_settings( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + source = json.dumps( + { + "smithy": "2.0", + "shapes": { + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + }, + "example#Input": {"type": "structure"}, + "example#Output": {"type": "structure"}, + "example#Selected": {"type": "structure"}, + "example#Excluded": {"type": "structure"}, + }, + } + ).encode() + selected = tmp_path / "selected" + assert ( + _run_types( + selected, + "--selector", + "[id|name=Selected]", + source=source, + ) + == 0 + ) + stub = (selected / "src/example_types/models.pyi").read_text() + assert "class Selected(" in stub + assert "class Excluded(" not in stub + assert "class Input(" not in stub + assert "class Output(" not in stub + + complete = tmp_path / "complete" + assert ( + _run_types( + complete, + "--generate-inputs-and-outputs", + source=source, + ) + == 0 + ) + stub = (complete / "src/example_types/models.pyi").read_text() + assert "class Input(" in stub + assert "class Output(" in stub + assert capsys.readouterr().err == "" + + +def test_types_generation_packages_dotted_modules_from_namespace_root( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + output = tmp_path / "output" + assert ( + _run_types( + output, + package="generated.nested_types", + version="1.0.0", + ) + == 0 + ) + project = tomllib.loads((output / "pyproject.toml").read_text()) + wheel = project["tool"]["hatch"]["build"]["targets"]["wheel"] + assert wheel["packages"] == ["src/generated"] + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("generated.nested_types") + assert package.__version__ == "1.0.0" + assert capsys.readouterr().err == "" @pytest.mark.parametrize( @@ -127,26 +288,73 @@ def test_invalid_model_is_a_model_failure( model = tmp_path / "model.json" model.write_bytes(source) - assert ( - main( - ( - "generate", - "types", - "--model", - str(model), - "--output", - str(tmp_path / "output"), - ), - environ={}, - ) - == 1 - ) + assert _run_types(tmp_path / "output", "--model", str(model)) == 1 err = capsys.readouterr().err assert err.startswith("smithy-python: error: ") assert expected in err assert "not implemented" not in err +@pytest.mark.parametrize("option", ["--package", "--package-version"]) +def test_types_generation_requires_package_settings( + option: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) + del args[index : index + 2] + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert f"the following arguments are required: {option}" in capsys.readouterr().err + assert not output.exists() + + +@pytest.mark.parametrize( + ("option", "value", "expected"), + [ + ( + "--package", + "invalid-package", + "Invalid Python package or distribution name", + ), + ("--package", "_example", "Invalid Python package or distribution name"), + ("--package", "example_", "Invalid Python package or distribution name"), + ("--package", "café", "Invalid Python package or distribution name"), + ("--package-version", "", "expected a non-empty value"), + ("--package-version", "not-a-version", "Invalid version"), + ("--selector", "[", "Expected an attribute path segment"), + ], +) +def test_types_generation_rejects_invalid_settings( + option: str, + value: str, + expected: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) if option in args else len(args) + if option in args: + args[index + 1] = value + else: + args.extend((option, value)) + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert expected in capsys.readouterr().err + assert not output.exists() + + +def test_types_generation_normalizes_package_version(tmp_path: Path) -> None: + output = tmp_path / "output" + + assert _run_types(output, version="v1.2") == 0 + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"]["version"] == "1.2" + + @pytest.mark.parametrize("option", ["--model", "--output"]) def test_run_plugin_rejects_direct_invocation_options( option: str, diff --git a/packages/smithy-python/tests/unit/test_selection.py b/packages/smithy-python/tests/unit/test_selection.py index a41eaf87e..b24bbeba0 100644 --- a/packages/smithy-python/tests/unit/test_selection.py +++ b/packages/smithy-python/tests/unit/test_selection.py @@ -10,6 +10,16 @@ import pytest from smithy_python.cli import main +_TYPES_OPTIONS = ("--package", "example_types", "--package-version", "1.0.0") + + +def _artifact_args(artifact: str, *options: str) -> list[str]: + args = ["generate", artifact] + if artifact == "types": + args.extend(_TYPES_OPTIONS) + args.extend(options) + return args + @pytest.mark.parametrize("artifact", ["client", "types"]) @pytest.mark.parametrize( @@ -53,19 +63,29 @@ def test_cli_service_selection( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--output", str(output)] + argv = _artifact_args(artifact, "--output", str(output)) if service is not None: argv.extend(("--service", service)) + if artifact == "types" and service in (None, "example#Second"): + expected = 0 + diagnostic = ( + "" + if service is None + else "excluded 1 eligible data shape(s) outside the closure" + ) assert main(argv, environ={}, stdin=BytesIO(source)) == expected captured = capsys.readouterr() assert captured.out == "" - assert diagnostic in captured.err + if diagnostic: + assert diagnostic in captured.err + else: + assert captured.err == "" assert "unrecognized arguments" not in captured.err if service is None: assert "example#Base" not in captured.err if expected == 2: assert "not implemented" not in captured.err - assert not output.exists() + assert output.exists() is (expected == 0) @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -82,18 +102,22 @@ def test_cli_implicit_service( if has_service: shapes["example#Service"] = {"type": "service"} source = json.dumps({"smithy": "2.0", "shapes": shapes}).encode() - requires_service = artifact == "client" and not has_service - assert main( - ("generate", artifact), - environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, - stdin=BytesIO(source), - ) == (2 if requires_service else 1) + argv = _artifact_args(artifact) + expected = 0 if artifact == "types" else 1 if has_service else 2 + assert ( + main( + argv, + environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, + stdin=BytesIO(source), + ) + == expected + ) captured = capsys.readouterr() assert captured.out == "" - assert ("not implemented" in captured.err) is not requires_service - if requires_service: + assert ("not implemented" in captured.err) is (artifact == "client" and has_service) + if artifact == "client" and not has_service: assert "service" in captured.err - assert not (tmp_path / "output").exists() + assert (tmp_path / "output").exists() is (artifact == "types") @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -124,19 +148,24 @@ def test_cli_reports_only_excluded_eligible_data( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--service", "example#Service"] + argv = _artifact_args(artifact, "--service", "example#Service") environ = {"SMITHY_PLUGIN_DIR": str(output)} if plugin else {} if not plugin: argv.extend(("--output", str(output))) - assert main(argv, environ=environ, stdin=BytesIO(source)) == 1 + assert main(argv, environ=environ, stdin=BytesIO(source)) == ( + 0 if artifact == "types" else 1 + ) captured = capsys.readouterr() assert captured.out == "" - assert captured.err == ( - "smithy-python: excluded 1 eligible data shape(s) " - "outside the closure of service example#Service\n" - f"smithy-python: error: {artifact} generation is not implemented yet\n" - ) - assert not output.exists() + expected = "" + if artifact == "client": + expected = ( + "smithy-python: excluded 1 eligible data shape(s) " + "outside the closure of service example#Service\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + assert captured.err == expected + assert output.exists() is (artifact == "types") def test_cli_conflicts_are_model_errors( @@ -146,7 +175,7 @@ def test_cli_conflicts_are_model_errors( { "smithy": "2.0", "shapes": { - shape_id: {"type": "string"} + shape_id: {"type": "structure"} for shape_id in ("a#Name", "b#name", "c#NAME", "a#Other", "b#other") }, } @@ -154,7 +183,7 @@ def test_cli_conflicts_are_model_errors( output = tmp_path / "output" assert ( main( - ("generate", "types", "--output", str(output)), + _artifact_args("types", "--output", str(output)), environ={}, stdin=BytesIO(source), ) diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 3fb2e4a4a..fbce403cb 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -514,6 +514,9 @@ def test_only_emitted_declarations_collide() -> None: "pkg.class", "None", "pkg/child", + "_example", + "example_", + "café", "pkg.\uff43\uff4c\uff41\uff53\uff53", ], ) diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index b84b7ef58..b904a65cf 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1700,6 +1700,9 @@ "SymbolProvider": "class", "TypeReference": "class" }, + "smithy_python.types": { + "TypesGenerator": "class" + }, "smithy_python.writer": { "CodegenError": "rex-exception", "PythonWriter": "class", diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 4306ffda4..a341e9a95 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -9,14 +9,7 @@ import sys from pathlib import Path -from smithy_python.dependencies import DependencySet -from smithy_python.files import FileManifest -from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings -from smithy_python.model import load_model -from smithy_python.models import ModelGenerator, _model_reserved_member_names -from smithy_python.schemas import SchemaGenerator -from smithy_python.selection import select_type_shapes -from smithy_python.symbols import SymbolProvider +from smithy_python.cli import main as generate _ROOT = Path(__file__).resolve().parents[2] _MODEL = Path(__file__).with_name("model.json") @@ -26,36 +19,28 @@ def main() -> None: - model = load_model(_MODEL.read_bytes()) - selection = select_type_shapes( - model, - service_id=_SERVICE, - generate_inputs_and_outputs=True, + result = generate( + ( + "generate", + "types", + "--model", + str(_MODEL), + "--output", + str(_OUTPUT), + "--service", + _SERVICE, + "--package", + _PACKAGE, + "--package-version", + "1.0.0", + "--package-description", + "Example Python codegen output", + "--generate-inputs-and-outputs", + ), + environ={}, ) - settings = TypesSettings( - package=_PACKAGE, - package_version="1.0.0", - package_description="Example Python codegen output", - ) - symbols = SymbolProvider( - model, - selection, - package=settings.package, - reserved_member_names=_model_reserved_member_names(model), - ) - dependencies = DependencySet() - context = GenerationContext( - model=model, - selection=selection, - settings=settings, - symbols=symbols, - plan=GenerationPlan.from_selection(model, selection, symbols), - dependencies=dependencies, - files=FileManifest(dependencies), - ) - SchemaGenerator().generate(context) - ModelGenerator().generate(context) - context.files.commit(_OUTPUT) + if result: + raise SystemExit(result) _format_output() diff --git a/uv.lock b/uv.lock index 4cbf599d5..f7f327293 100644 --- a/uv.lock +++ b/uv.lock @@ -771,6 +771,12 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "smithy-python" source = { editable = "packages/smithy-python" } +dependencies = [ + { name = "packaging" }, +] + +[package.metadata] +requires-dist = [{ name = "packaging" }] [[package]] name = "smithy-python-workspace" From fc2127c6cf501637dc445ceee4d0d64763a07bd3 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:14:59 -0400 Subject: [PATCH 3/5] rename Python codegen output labels --- .github/workflows/ci.yml | 2 +- codegen-output-py/types/.smithy-python-manifest.json | 10 +++++----- codegen-output-py/types/README.md | 2 +- codegen-output-py/types/pyproject.toml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a257293af..ea935848c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: run: | make build-py - - name: Check native Python codegen output + - name: Check Python codegen output if: ${{ matrix.python-version == '3.12' }} run: | make generate-codegen-output-py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 01fe60f9b..9137da1c2 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,12 +1,12 @@ { "files": { - "README.md": "03146affd4cdec7675387b787ff852400641bad4c0cfc5e80bb860d576acee4b", - "pyproject.toml": "8b49dfb34a75d368f1c264a8369fb4093af3b9618ec4a93173b28315578c808e", + "README.md": "b070865ebb1dc46192ea8a45f0961e1b0f812dfccd069537cfc04311b3327660", + "pyproject.toml": "c419d5bcb7e49329ae50cab3b1a766446d3d14d40a422328788f93e8b2bbeb2b", "src/example_types/__init__.py": "6d153749d98df8adfe6264fb74d642917cbf2d6922b9211719dee35483dab943", "src/example_types/_private/__init__.py": "5384bfdb2df380b6557cc7a71d16891415bccaa87699406e236f752c6415389f", - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", - "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", - "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14", + "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", + "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", + "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a", "src/example_types/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "version": 1 diff --git a/codegen-output-py/types/README.md b/codegen-output-py/types/README.md index a448afa11..8e6571146 100644 --- a/codegen-output-py/types/README.md +++ b/codegen-output-py/types/README.md @@ -2,4 +2,4 @@ ## example_types Types Package -Example native Python codegen output +Example Python codegen output diff --git a/codegen-output-py/types/pyproject.toml b/codegen-output-py/types/pyproject.toml index ac59b5076..5df7d40fb 100644 --- a/codegen-output-py/types/pyproject.toml +++ b/codegen-output-py/types/pyproject.toml @@ -3,7 +3,7 @@ [project] name = "example_types" version = "1.0.0" -description = "Example native Python codegen output" +description = "Example Python codegen output" readme = "README.md" requires-python = ">=3.12" keywords = ["smithy", "example_types"] From 7a227d6e639f458509a9b6612b6246d796b85411 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:40:21 -0400 Subject: [PATCH 4/5] make generated annotations and package validation explicit --- .../types/.smithy-python-manifest.json | 12 +++---- codegen-output-py/types/pyproject.toml | 7 ---- .../types/src/example_types/__init__.py | 2 -- .../src/example_types/_private/__init__.py | 1 - .../src/example_types/_private/schemas.py | 2 -- .../types/src/example_types/models.py | 4 +-- .../types/src/example_types/models.pyi | 2 -- designs/codegen/types.md | 13 ++++---- designs/codegen/writer.md | 33 +++++++++++-------- packages/smithy-python/pyproject.toml | 4 +-- .../smithy-python/src/smithy_python/cli.py | 23 ++++++++----- .../smithy-python/src/smithy_python/files.py | 11 ++++++- .../smithy-python/src/smithy_python/models.py | 6 +++- .../smithy-python/src/smithy_python/types.py | 7 ---- .../smithy-python/src/smithy_python/writer.py | 18 +++++++--- packages/smithy-python/tests/unit/test_cli.py | 4 +-- .../smithy-python/tests/unit/test_files.py | 23 +++++++++++-- .../smithy-python/tests/unit/test_writer.py | 17 ++++++---- scripts/codegen-output-py/generate.py | 2 +- uv.lock | 6 ---- 20 files changed, 112 insertions(+), 85 deletions(-) diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9137da1c2..8daa8d4cd 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,12 +1,12 @@ { "files": { "README.md": "b070865ebb1dc46192ea8a45f0961e1b0f812dfccd069537cfc04311b3327660", - "pyproject.toml": "c419d5bcb7e49329ae50cab3b1a766446d3d14d40a422328788f93e8b2bbeb2b", - "src/example_types/__init__.py": "6d153749d98df8adfe6264fb74d642917cbf2d6922b9211719dee35483dab943", - "src/example_types/_private/__init__.py": "5384bfdb2df380b6557cc7a71d16891415bccaa87699406e236f752c6415389f", - "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", - "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", - "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a", + "pyproject.toml": "3b4edeff9fe5c4641d8729e49abfa1e7c1936484d0512ad273a3724ebb54f40a", + "src/example_types/__init__.py": "27ea4bc5ba93a6fc6e1f11e9784565de24e9249f0d3e77a9e2fb64fa47d11012", + "src/example_types/_private/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/example_types/_private/schemas.py": "b3c1dd3b1ab8511ef45cca22cf415726aec599edfd457b30cdb22482bcd5b799", + "src/example_types/models.py": "262a8c40dce4fb282aca6b17214296db0c4fc1da526a50a3155cde0fe7344b6b", + "src/example_types/models.pyi": "36be0dc7d1d9402394e531d782fab2392d55cc60a949a1bcae4c4f469de6cc11", "src/example_types/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "version": 1 diff --git a/codegen-output-py/types/pyproject.toml b/codegen-output-py/types/pyproject.toml index 5df7d40fb..4df29b3b4 100644 --- a/codegen-output-py/types/pyproject.toml +++ b/codegen-output-py/types/pyproject.toml @@ -25,13 +25,6 @@ packages = [ [tool.pyright] typeCheckingMode = "strict" -reportPrivateUsage = false -reportUnusedFunction = false -reportUnusedVariable = false -reportUnnecessaryComparison = false -reportUnusedClass = false -enableExperimentalFeatures = true -exclude = ["**/models.py"] [tool.ruff] target-version = "py312" diff --git a/codegen-output-py/types/src/example_types/__init__.py b/codegen-output-py/types/src/example_types/__init__.py index f81c022be..3e7d37b5b 100644 --- a/codegen-output-py/types/src/example_types/__init__.py +++ b/codegen-output-py/types/src/example_types/__init__.py @@ -1,3 +1 @@ -from __future__ import annotations - __version__: str = "1.0.0" diff --git a/codegen-output-py/types/src/example_types/_private/__init__.py b/codegen-output-py/types/src/example_types/_private/__init__.py index 9d48db4f9..e69de29bb 100644 --- a/codegen-output-py/types/src/example_types/_private/__init__.py +++ b/codegen-output-py/types/src/example_types/_private/__init__.py @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/codegen-output-py/types/src/example_types/_private/schemas.py b/codegen-output-py/types/src/example_types/_private/schemas.py index 91264ec48..c2c99e18e 100644 --- a/codegen-output-py/types/src/example_types/_private/schemas.py +++ b/codegen-output-py/types/src/example_types/_private/schemas.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from smithy_core._schema_compact import StaticSchema n0 = "example" diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py index 6803f9606..6ef4b985d 100644 --- a/codegen-output-py/types/src/example_types/models.py +++ b/codegen-output-py/types/src/example_types/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from smithy_core._dynamic_models import ModelRegistry from smithy_core.exceptions import ModeledError @@ -43,7 +41,7 @@ class ServiceError(ModeledError): } _MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") -__all__ = ("ServiceError", *_MODEL_NAMES) +__all__ = ("ServiceError", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll] _registry = ModelRegistry( diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi index 0d72eca2a..6502961ba 100644 --- a/codegen-output-py/types/src/example_types/models.pyi +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -1,5 +1,3 @@ -from __future__ import annotations - from dataclasses import dataclass from enum import IntEnum, StrEnum from typing import Self diff --git a/designs/codegen/types.md b/designs/codegen/types.md index 9f7135dc3..58e07fa24 100644 --- a/designs/codegen/types.md +++ b/designs/codegen/types.md @@ -112,7 +112,7 @@ The `types` command accepts: * `--package-version` * `--package-description` * `--selector` -* `--generate-inputs-and-outputs` +* `--include-operation-io` `package` and `package-version` are required. The package identifies the generated import package and follows the validation rules in `SymbolProvider`. @@ -129,7 +129,7 @@ Types generation never infers a service. The options apply in this order: 2. `--selector` narrows those declaration shapes. The default identity selector keeps every candidate. 3. Input and output structures are removed as roots unless - `--generate-inputs-and-outputs` is set. + `--include-operation-io` is set. 4. Member-target dependencies of the remaining declarations are added transitively. @@ -382,16 +382,17 @@ model names: ```python _MODEL_NAMES = tuple(_MODEL_INDEX) -__all__ = ("ServiceError", *_MODEL_NAMES) +__all__ = ("ServiceError", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll] def __dir__() -> list[str]: return sorted({*globals(), *__all__}) ``` -This keeps Ruff from reporting `F822` for lazy exports. Star imports read -`__all__` and resolve each model through module `__getattr__`. Computing -`__dir__` from the index does not construct model classes. +This keeps Ruff from reporting `F822` for lazy exports. The targeted Pyright +suppression replaces excluding the complete runtime module from analysis. Star +imports read `__all__` and resolve each model through module `__getattr__`. +Computing `__dir__` from the index does not construct model classes. Model member access uses the schema's dense member indexes where positional dispatch is required. Name maps remain dictionaries where lookup is by a wire diff --git a/designs/codegen/writer.md b/designs/codegen/writer.md index 1c581d1d1..bfd140d1d 100644 --- a/designs/codegen/writer.md +++ b/designs/codegen/writer.md @@ -11,7 +11,7 @@ from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter node = TypeReference("Node", "example.models", nullable=True) -writer = PythonWriter("example.models") +writer = PythonWriter("example.models", future_annotations=True) writer.line("class Node:") with writer.indent(): writer.line("children: ", TypeReference("list", "builtins", (node,))) @@ -34,8 +34,9 @@ class Node: ## Writing a module -`PythonWriter(module, dependencies=None)` takes the destination module name and -an optional shared `DependencySet`. `render()` finds the names bound by the +`PythonWriter(module, dependencies=None, future_annotations=False)` takes the +destination module name, an optional shared `DependencySet`, and whether to emit +`from __future__ import annotations`. `render()` finds the names bound by the written lines and follows Python's scoping rules. Module-level names, such as classes, functions, assignments, imports and `global` assignments, apply to every reference. A local name, such as a field, @@ -56,9 +57,11 @@ symbol provider ensures that declaration and field names are unique. current indentation. `line()` writes a blank line without spaces. * `indent()` adds four spaces inside a context manager and restores indentation even when the block raises an exception. -* `render()` returns source ending in a newline, with a future-annotations - header, sorted imports and the body in written order. It does not modify the - writer. Adding another reference can change aliases in the next render. +* `add_future_annotations()` requests deferred annotation evaluation. +* `render()` returns source ending in a newline, with an optional + future-annotations header, sorted imports and the body in written order. It + does not modify the writer. Adding another reference can change aliases in the + next render. Raw text is trusted Python, not a template. To find names, `render()` parses the written lines with the Python 3.12 grammar, with a unique placeholder @@ -104,9 +107,9 @@ level: `list[str | None]` differs from `list[str] | None`. The `None` literal stays `None`, including when marked nullable. Only that literal can omit its module. Other module-less references raise `CodegenError`. -Generated source targets Python 3.12+. The future import permits references to -classes defined later, including recursive types. These are annotations, not -expressions to evaluate while defining classes. +Generated source targets Python 3.12+. When requested, the future import permits +references to classes defined later, including recursive types. These are +annotations, not expressions to evaluate while defining classes. * Builtins normally use `str`, `int`, `list[T]` and `dict[K, V]`. * Types in the current module use their bare names without imports. @@ -130,14 +133,16 @@ metadata. Annotation-only references still contribute dependencies because type checking a generated package must resolve them. Writers created by `FileManifest.python_module()` share the generation -context's dependency set. A standalone writer without a dependency set renders -without recording dependencies. +context's dependency set. Passing `future_annotations=True` requests the future +import even when another contributor already created the shared writer. A +standalone writer without a dependency set renders without recording +dependencies. ## Names that collide -An import uses a module-derived alias when its name matches a module-level -name, a local name visible where it is referenced, a builtin, the future -import's `annotations` name, or another imported type. +An import uses a module-derived alias when its name matches a module-level name, +a local name visible where it is referenced, a builtin, the future import's +`annotations` name when enabled, or another imported type. For example, `decimal.Decimal` becomes `_decimal_Decimal`. Dots in a module path become underscores. All imports sharing a short name receive aliases, regardless of the order they were written. diff --git a/packages/smithy-python/pyproject.toml b/packages/smithy-python/pyproject.toml index 2a3e2ec9e..cab33c833 100644 --- a/packages/smithy-python/pyproject.toml +++ b/packages/smithy-python/pyproject.toml @@ -26,9 +26,7 @@ classifiers = [ "Programming Language :: Python :: Free Threading :: 2 - Beta", "Topic :: Software Development :: Code Generators", ] -dependencies = [ - "packaging", -] +dependencies = [] [project.scripts] smithy-python = "smithy_python.cli:main" diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index 15f8012ab..5edb2eda4 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -5,14 +5,13 @@ from __future__ import annotations import argparse +import re import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import BinaryIO, Final -from packaging.version import InvalidVersion, Version - from . import __version__ from . import dependencies as _dependencies from . import exceptions as _exceptions @@ -33,6 +32,14 @@ _GENERATION_NOT_IMPLEMENTED: Final = ( "smithy-python: error: {artifact} generation is not implemented yet\n" ) +_PACKAGE_VERSION: Final = re.compile( + r"v?[0-9]+(?:\.[0-9]+)*" + r"(?:(?:a|b|rc)[0-9]+)?" + r"(?:\.post[0-9]+)?" + r"(?:\.dev[0-9]+)?" + r"(?:\+[a-z0-9]+(?:[._-][a-z0-9]+)*)?", + re.IGNORECASE, +) @dataclass(frozen=True, slots=True) @@ -125,7 +132,8 @@ def _create_parser() -> argparse.ArgumentParser: help="Smithy selector used to narrow generated declaration roots.", ) types.add_argument( - "--generate-inputs-and-outputs", + "--include-operation-io", + dest="generate_inputs_and_outputs", action="store_true", help="Include operation input and output structures as generation roots.", ) @@ -265,9 +273,6 @@ def _parse_selector(value: str) -> _selection.Selector: def _package_version(value: str) -> str: - if not value: - raise argparse.ArgumentTypeError("expected a non-empty value") - try: - return str(Version(value)) - except InvalidVersion as error: - raise argparse.ArgumentTypeError(str(error)) from error + if _PACKAGE_VERSION.fullmatch(value) is None: + raise argparse.ArgumentTypeError(f"Invalid version: {value!r}") + return value[1:] if value[:1].lower() == "v" else value diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py index 79fa17069..2e015a27e 100644 --- a/packages/smithy-python/src/smithy_python/files.py +++ b/packages/smithy-python/src/smithy_python/files.py @@ -37,7 +37,13 @@ def dependencies(self) -> _dependencies.DependencySet: """Dependencies recorded by writers owned by this manifest.""" return self._dependencies - def python_module(self, path: str, module: str) -> _writer.PythonWriter: + def python_module( + self, + path: str, + module: str, + *, + future_annotations: bool = False, + ) -> _writer.PythonWriter: """Return the shared writer for a generated Python module.""" path = _normalize_path(path) self._claim_public_path(path) @@ -48,6 +54,8 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: f"Generated path {path!r} already uses module " f"{existing.module!r}, not {module!r}" ) + if future_annotations: + existing.writer.add_future_annotations() return existing.writer if existing is not None: raise _exceptions.CodegenError( @@ -56,6 +64,7 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: writer = _writer.PythonWriter( module, dependencies=self._dependencies, + future_annotations=future_annotations, stub=path.endswith(".pyi"), ) self._files[path] = _PythonFile(module, writer) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py index 0e02263a0..29be7e6ea 100644 --- a/packages/smithy-python/src/smithy_python/models.py +++ b/packages/smithy-python/src/smithy_python/models.py @@ -129,7 +129,11 @@ def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: ) else: writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") - writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line( + "__all__ = (", + _quote(exports[0]), + ", *_MODEL_NAMES) # pyright: ignore[reportUnsupportedDunderAll]", + ) writer.line() writer.line() writer.line("_registry = ", _MODEL_REGISTRY, "(") diff --git a/packages/smithy-python/src/smithy_python/types.py b/packages/smithy-python/src/smithy_python/types.py index ef2b43560..5ab58acd8 100644 --- a/packages/smithy-python/src/smithy_python/types.py +++ b/packages/smithy-python/src/smithy_python/types.py @@ -34,13 +34,6 @@ def generate( _PYPROJECT_SUFFIX = """ [tool.pyright] typeCheckingMode = "strict" -reportPrivateUsage = false -reportUnusedFunction = false -reportUnusedVariable = false -reportUnnecessaryComparison = false -reportUnusedClass = false -enableExperimentalFeatures = true -exclude = ["**/models.py"] [tool.ruff] target-version = "py312" diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index efcf45e4b..92bea66f5 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -62,15 +62,21 @@ def __init__( module: str, *, dependencies: _dependencies.DependencySet | None = None, + future_annotations: bool = False, stub: bool = False, ) -> None: _validate_module(module, relative=False) self._module = _binding(module) self._dependencies = dependencies + self._future_annotations = future_annotations self._stub = stub self._depth = 0 self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] + def add_future_annotations(self) -> None: + """Include ``from __future__ import annotations`` in rendered source.""" + self._future_annotations = True + def line(self, *parts: str | TypeReference) -> None: """Append one line, concatenating its parts without separators.""" if self._dependencies is not None: @@ -94,12 +100,16 @@ def indent(self) -> Generator[None]: def render(self) -> str: """Return complete source without changing the recorded lines.""" names, imports = self._plan_imports(*self._names()) - header = "from __future__ import annotations\n" + header = ( + "from __future__ import annotations\n" if self._future_annotations else "" + ) if imports: - header += "\n" + _render_imports(imports) + "\n" + header += ("\n" if header else "") + _render_imports(imports) + "\n" body = self._body(lambda reference: self._annotation(reference, names)) separator = "\n" if self._stub else "\n\n" - return header + (separator + "\n".join(body) + "\n" if body else "") + return header + ( + (separator if header else "") + "\n".join(body) + "\n" if body else "" + ) def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -199,7 +209,7 @@ def _plan_imports( if binding in reserved or _is_local(binding, scoped) or binding in _BUILTINS - or binding == "annotations" + or (self._future_annotations and binding == "annotations") or counts[binding] > 1 else name ) diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index a0d554608..f468a4ed7 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -186,7 +186,7 @@ def test_types_generation_applies_selector_and_input_output_settings( assert ( _run_types( complete, - "--generate-inputs-and-outputs", + "--include-operation-io", source=source, ) == 0 @@ -322,7 +322,7 @@ def test_types_generation_requires_package_settings( ("--package", "_example", "Invalid Python package or distribution name"), ("--package", "example_", "Invalid Python package or distribution name"), ("--package", "café", "Invalid Python package or distribution name"), - ("--package-version", "", "expected a non-empty value"), + ("--package-version", "", "Invalid version"), ("--package-version", "not-a-version", "Invalid version"), ("--selector", "[", "Expected an attribute path segment"), ], diff --git a/packages/smithy-python/tests/unit/test_files.py b/packages/smithy-python/tests/unit/test_files.py index 098d0e54f..b84e04bbe 100644 --- a/packages/smithy-python/tests/unit/test_files.py +++ b/packages/smithy-python/tests/unit/test_files.py @@ -43,8 +43,7 @@ def test_file_manifest_reuses_python_writer_and_records_dependencies( assert ( output.joinpath("example/models.py").read_text(encoding="utf-8") - == "from __future__ import annotations\n\n" - "from smithy_core.documents import Document\n\n\n" + == "from smithy_core.documents import Document\n\n\n" "value: list[Document]\n" ) assert output.joinpath("example/py.typed").read_text(encoding="utf-8") == "" @@ -93,6 +92,26 @@ def test_file_manifest_rejects_conflicting_claims(tmp_path: Path) -> None: files.commit(tmp_path / "output") +def test_file_manifest_combines_future_annotation_requests(tmp_path: Path) -> None: + files = FileManifest() + writer = files.python_module("models.py", "example.models") + assert writer is files.python_module( + "models.py", "example.models", future_annotations=True + ) + writer.line("class Node:") + with writer.indent(): + writer.line("child: ", TypeReference("Node", "example.models")) + + output = tmp_path / "output" + files.commit(output) + + assert ( + output.joinpath("models.py") + .read_text() + .startswith("from __future__ import annotations\n") + ) + + def test_validation_failure_leaves_existing_output_untouched(tmp_path: Path) -> None: output = tmp_path / "output" output.mkdir() diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index c7df26470..3d6f7b39d 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -71,7 +71,7 @@ def load_written_module( def test_nested_forward_and_mutual_annotations( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - writer = PythonWriter("written_models") + writer = PythonWriter("written_models", future_annotations=True) writer.line("class Node:") with writer.indent(): writer.line( @@ -607,7 +607,7 @@ def test_builtin_names_cover_running_interpreter() -> None: def test_equivalent_current_module_names( module: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - writer = PythonWriter(module) + writer = PythonWriter(module, future_annotations=True) writer.line("class Node:") with writer.indent(): writer.line( @@ -685,13 +685,20 @@ def test_deep_collection_rendering() -> None: def test_future_import_name_is_reserved() -> None: - writer = PythonWriter("models") + writer = PythonWriter("models", future_annotations=True) writer.line("value: ", TypeReference("annotations", "external")) assert ( "from external import annotations as _external_annotations" in writer.render() ) +def test_future_annotations_are_opt_in() -> None: + assert PythonWriter("models").render() == "" + assert PythonWriter("models", future_annotations=True).render() == ( + "from __future__ import annotations\n" + ) + + @pytest.mark.parametrize( "reference", [ @@ -767,7 +774,7 @@ def test_generation_without_runtime_packages() -> None: def test_empty_writer_and_trusted_text() -> None: writer = PythonWriter("models") - assert writer.render() == "from __future__ import annotations\n" + assert writer.render() == "" writer.line("value = '$T {not_a_template}'") assert writer.render().endswith("value = '$T {not_a_template}'\n") @@ -806,8 +813,6 @@ def test_imports_are_grouped_and_combined() -> None: writer.line("fourth: ", TypeReference("Thing", "._private")) assert writer.render().startswith( - "from __future__ import annotations\n" - "\n" "from enum import IntEnum, StrEnum\n" "\n" "from smithy_core.documents import Document\n" diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index a341e9a95..991806d4d 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -35,7 +35,7 @@ def main() -> None: "1.0.0", "--package-description", "Example Python codegen output", - "--generate-inputs-and-outputs", + "--include-operation-io", ), environ={}, ) diff --git a/uv.lock b/uv.lock index f7f327293..4cbf599d5 100644 --- a/uv.lock +++ b/uv.lock @@ -771,12 +771,6 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "smithy-python" source = { editable = "packages/smithy-python" } -dependencies = [ - { name = "packaging" }, -] - -[package.metadata] -requires-dist = [{ name = "packaging" }] [[package]] name = "smithy-python-workspace" From fe17013b872d7f4f152c2d762c2950e45a7d11d0 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:57:01 -0400 Subject: [PATCH 5/5] partition schemas and model stubs --- designs/codegen/index.md | 1 + designs/codegen/partitioning.md | 163 +++++++++++ designs/codegen/types.md | 28 +- .../src/smithy_python/generation.py | 9 + .../smithy-python/src/smithy_python/models.py | 124 ++++++++- .../src/smithy_python/schemas.py | 260 ++++++++++++++++-- .../tests/unit/test_generation.py | 9 + .../smithy-python/tests/unit/test_models.py | 85 +++++- .../smithy-python/tests/unit/test_schemas.py | 167 ++++++++++- 9 files changed, 792 insertions(+), 54 deletions(-) create mode 100644 designs/codegen/partitioning.md diff --git a/designs/codegen/index.md b/designs/codegen/index.md index 5455a37b9..81d968097 100644 --- a/designs/codegen/index.md +++ b/designs/codegen/index.md @@ -65,3 +65,4 @@ behavior of generated packages. * [Python Symbols](symbols.md) * [Python Writer](writer.md) * [Standalone Types Generation](types.md) +* [Code Generation Partitioning](partitioning.md) diff --git a/designs/codegen/partitioning.md b/designs/codegen/partitioning.md new file mode 100644 index 000000000..e4f0c8266 --- /dev/null +++ b/designs/codegen/partitioning.md @@ -0,0 +1,163 @@ +# Code Generation Partitioning + +## Abstract + +Large standalone types packages split compact schemas and model stubs across +private modules. Stable facade modules preserve generated imports while +reducing generator memory and static analysis time. + +## Motivation + +An EC2 types package contains 4,065 selected shapes. Emitting its schemas and +model declarations as two large files makes each file the unit of work for +rendering, staging, parsing, and static analysis. + +`FileManifest` releases each rendered file after staging it, so the largest file +sets the lower bound for source memory during publication. Pyright analyzes the +model stub as one large declaration file. Lazy schema hydration and model +construction reduce runtime work but do not reduce these file-level costs. + +Private partitions bound the work associated with one generated file. Stable +facades preserve generated imports and lazy runtime behavior while tools process +the smaller partition files. + +## Settings + +`TypesSettings` owns both partition targets: + +```python +@dataclass(frozen=True, slots=True) +class TypesSettings(ArtifactSettings): + selector: Selector = field(default_factory=Selector.identity) + generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 +``` + +The CLI uses these defaults. Python callers may set either target to a positive +integer or `None`. A value of `None` emits that surface as one file. + +Partitioning starts when the number of planned items exceeds its target. A +package with 250 schema shapes and 250 model declarations keeps the single-file +layout. + +## Output Layout + +Partitioned packages retain the stable module paths used by unpartitioned +packages. + +| Surface | Facade | Partitions | +| --- | --- | --- | +| Compact schemas | `._private.schemas` | `._private.schemas_N` | +| Model stubs | `.models.pyi` | `._private.models_N.pyi` | +| Runtime models | `.models` | Single module | + +### Schema File Layout + +```text +src// +├── models.py +└── _private/ + ├── __init__.py + ├── schemas.py + ├── schemas_0.py + ├── schemas_1.py + └── schemas_N.py +``` + +`models.py` imports the `schemas.py` facade. The facade maps schema constants to +partition indexes and loads the matching `schemas_N.py` module on first access. +Each partition contains compact schema tuples. + +### Stub File Layout + +```text +src// +├── models.py +├── models.pyi +└── _private/ + ├── __init__.py + ├── models_0.pyi + ├── models_1.pyi + └── models_N.pyi +``` + +`models.pyi` is the public typing facade. It imports declarations from each +private stub partition. `models.py` remains the runtime module. + +`FileManifest` registers the facade and partition files before publication. +Its ownership rules remove stale, unmodified partition files after a model +shrinks or a target changes. + +Files under `._private` remain private implementation details. + +## Schema Partitions + +`SchemaPlan` provides schema shapes in deterministic emission order and groups +each recursive strongly connected component. A recursive group stays in one +partition so its schemas can use direct local references. + +The schema partitioner: + +1. Packs recursive groups in plan order using the configured shape target. +2. Uses the resulting partition count for the final layout. +3. Estimates the emitted source size of each group. +4. Selects contiguous boundaries that balance estimated source size. + +The estimate includes shape and member identifiers, schema references, and +trait values. It uses plan metadata instead of rendering schemas a second time. +The target determines the partition count. Final partitions may exceed the +target to balance source size, and recursive groups remain indivisible. + +The facade maps each schema constant to a partition index. Its `__getattr__` +imports the selected partition, reads the constant, and caches the result in +the facade module. `__dir__` includes every planned schema constant. + +Partition modules route cross-partition references through the facade. This +keeps dependencies lazy and avoids direct import cycles between partitions. +Hydration remains lazy through `smithy_core._schema_compact`. + +## Stub Partitions + +`ModelPlan` provides model declarations in deterministic order. The stub +partitioner writes contiguous groups up to the configured target. + +The public `models.pyi` facade imports every declaration from its private stub +partition and defines the same `__all__` as `models.py`. Private stubs reference +generated declarations through the public `models` module. This keeps their +type references independent of private partition boundaries. + +Runtime `models.py` remains one lazy registry module. Runtime model construction +and class identity stay unchanged. + +## Compatibility + +Partitioning preserves: + +* Public model module paths and runtime schema lookups. +* Model names, constructor signatures, and type annotations. +* Public model `__all__`, explicit imports, star imports, and `dir()` results. +* Lazy schema hydration and model construction. +* Schema identity, recursive references, and serde behavior. + +Partition indexes follow generation-plan order. Model changes can move +declarations between private files while the public facades remain stable. + +## Validation + +Tests cover packages below and above each target, recursive schema groups, +cross-partition references, facade caching, strict Pyright, Ruff, and +deterministic output. + +The final x86 EC2 benchmark compared unpartitioned output with 250-item schema +and stub partitions: + +| Metric | Unpartitioned | Partitioned | Change | +| --- | ---: | ---: | ---: | +| Generation time | 3.557 s | 3.555 s | -0.1% | +| Peak RSS | 264.5 MiB | 122.4 MiB | -53.7% | +| Peak Python allocation | 222.6 MiB | 84.7 MiB | -61.9% | +| Generated bytes | 4.78 MiB | 5.33 MiB | +11.6% | +| Model import | 50.6 ms | 40.4 ms | -20.2% | +| First model access | 12.3 ms | 22.3 ms | +10.0 ms | +| Strict Pyright | 81.98 s | 15.25 s | -81.4% | diff --git a/designs/codegen/types.md b/designs/codegen/types.md index 58e07fa24..964c4b6f8 100644 --- a/designs/codegen/types.md +++ b/designs/codegen/types.md @@ -100,11 +100,15 @@ class ArtifactSettings: class TypesSettings(ArtifactSettings): selector: Selector = field(default_factory=Selector.identity) generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 ``` The resolved service remains on `Selection`. It is not duplicated in settings. The CLI subcommand identifies the artifact, so settings do not duplicate that value. Artifact-specific settings extend the shared frozen dataclass. +Partition targets and their output layout are defined in +[Code Generation Partitioning](partitioning.md). The `types` command accepts: @@ -416,26 +420,12 @@ Package setup renders the dependency set collected during generation. ## Large Models -Schema and stub partitioning must not change public imports. The output API and -generation plan allow private content to move behind stable facade modules. +Large standalone types packages partition compact schemas and model stubs +behind stable facade modules. Schema partitions use source-size weighting and +keep recursive components together. Stub partitions use declaration counts. -smithy-java partitions schema classes after a model reaches 500 schema fields -and then targets about 50 KB of generated schema content per class. smithy-php -uses a centralized lazy schema registry. Python can combine those approaches: - -* Keep `._private.schemas` as the stable facade. -* Place compact schema tuples in private partition modules. -* Keep every recursive strongly connected component in one partition. -* Route schema constant access through generated indexes. -* Partition large type stubs behind the public `models.pyi` facade. - -Python partitioning thresholds must be based on Python measurements. Relevant inputs include writer render time, peak generator memory, generated import time, and type -checker time. - -Partitioning bounds the syntax tree analyzed by `PythonWriter` and prevents a -single EC2-sized source file from dominating codegen memory. Measurements -establish the partitioning thresholds. Facade modules preserve generated -imports. +[Code Generation Partitioning](partitioning.md) defines the output layout, +thresholds, and compatibility requirements. ## Performance diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index ce75e1e19..b82369040 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -44,6 +44,15 @@ class TypesSettings(ArtifactSettings): selector: _selection.Selector = field(default_factory=_selection.Selector.identity) generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 + + def __post_init__(self) -> None: + if any( + size is not None and size < 1 + for size in (self.schema_partition_size, self.stub_partition_size) + ): + raise ValueError("Partition sizes must be positive") @dataclass(frozen=True, slots=True) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py index 29be7e6ea..a9c3e8b57 100644 --- a/packages/smithy-python/src/smithy_python/models.py +++ b/packages/smithy-python/src/smithy_python/models.py @@ -36,12 +36,31 @@ def generate(self, context: GenerationContext[TypesSettings]) -> None: ), context.plan.models, ) - _StubEmitter( + facade = _StubEmitter( context.files.python_module( f"src/{directory}/models.pyi", f"{package}.models" ), context, - ).write() + ) + shapes = context.plan.models.shapes + size = context.settings.stub_partition_size + if size is None or len(shapes) <= size: + facade.write() + return + + partitions = tuple( + shapes[index : index + size] for index in range(0, len(shapes), size) + ) + facade.write_facade(partitions) + for index, shapes in enumerate(partitions): + _StubEmitter( + context.files.python_module( + f"src/{directory}/_private/models_{index}.pyi", + f"{package}._private.models_{index}", + ), + context, + external_bases=True, + ).write_partition(shapes) def _core_reference(name: str, module: str) -> _symbols.TypeReference: @@ -165,12 +184,25 @@ def __init__( self, writer: PythonWriter, context: GenerationContext[TypesSettings], + *, + external_bases: bool = False, ) -> None: self.writer = writer self.model = context.model self.plan = context.plan.models self.symbols = context.symbols self.module = f"{context.settings.package}.models" + self.external_bases = external_bases + self.serde_shape: str | _symbols.TypeReference = ( + _symbols.TypeReference("_SerdeShape", "..models") + if external_bases + else "_SerdeShape" + ) + self.service_error: str | _symbols.TypeReference = ( + _symbols.TypeReference("ServiceError", "..models") + if external_bases + else "ServiceError" + ) operations = tuple(context.model.iter_shapes(_model.OperationShape)) self.inputs = {operation.input for operation in operations} | { shape.id for shape in self.plan.shapes if shape.has_trait("input") @@ -180,6 +212,30 @@ def __init__( } def write(self) -> None: + """Write the complete unpartitioned public stub.""" + self._write_preamble() + self._write_shapes(self.plan.shapes) + self._write_unit() + + def write_facade(self, partitions: tuple[tuple[_model.Shape, ...], ...]) -> None: + """Write the public stub facade and re-export private declarations.""" + writer = self.writer + for index, shapes in enumerate(partitions): + writer.line(f"from ._private.models_{index} import (") + with writer.indent(): + for shape in shapes: + for name in _stub_shape_names(self.plan, shape): + writer.line(name, ",") + writer.line(")") + writer.line() + self._write_preamble() + self._write_unit() + + def write_partition(self, shapes: tuple[_model.Shape, ...]) -> None: + """Write declarations assigned to one private stub partition.""" + self._write_shapes(shapes) + + def _write_preamble(self) -> None: writer = self.writer writer.line("__all__ = [") with writer.indent(): @@ -212,7 +268,9 @@ def write(self) -> None: with writer.indent(): _write_docstring(writer, _SERVICE_ERROR_DOC) - for shape in self.plan.shapes: + def _write_shapes(self, shapes: Collection[_model.Shape]) -> None: + writer = self.writer + for shape in shapes: writer.line() match shape.type: case _model.ShapeType.STRUCTURE: @@ -223,10 +281,13 @@ def write(self) -> None: self._write_enum(shape) case _: pass + + def _write_unit(self) -> None: + writer = self.writer if self.plan.includes_unit: writer.line() writer.line("@", _DATACLASS, "(kw_only=True)") - writer.line("class Unit(_SerdeShape):") + writer.line("class Unit(", self.serde_shape, "):") with writer.indent(): writer.line("pass") @@ -235,10 +296,17 @@ def _write_structure(self, shape: _model.Shape) -> None: name = self.plan.declaration_names[shape.id] error = shape.has_trait("error") writer.line("@", _DATACLASS, "(kw_only=True)") + bases: tuple[str | _symbols.TypeReference, ...] = ( + (self.service_error, ", ", self.serde_shape) + if error + else (self.serde_shape,) + ) writer.line( "class ", name, - "(ServiceError, _SerdeShape):" if error else "(_SerdeShape):", + "(", + *bases, + "):", ) with writer.indent(): members = tuple( @@ -255,7 +323,7 @@ def _write_structure(self, shape: _model.Shape) -> None: if members or carries_metadata: writer.line() for member in members: - reference = self.symbols.type_reference(member.target) + reference = self._type_reference(member.target) has_default = member.has_trait("default") or self.model.get_target( member ).has_trait("default") @@ -283,20 +351,20 @@ def _write_union(self, shape: _model.Shape) -> None: variants: list[_symbols.TypeReference] = [] for member in shape.members.values(): name = self.plan.union_variant_names[member.id] - variants.append(_symbols.TypeReference(name, self.module)) + variants.append(self._public_reference(name)) writer.line("@", _DATACLASS) - writer.line("class ", name, "(_SerdeShape):") + writer.line("class ", name, "(", self.serde_shape, "):") with writer.indent(): if docs := _documentation(member): _write_docstring(writer, docs) writer.line() - writer.line("value: ", self.symbols.type_reference(member.target)) + writer.line("value: ", self._type_reference(member.target)) writer.line() unknown = self.plan.union_unknown_names[shape.id] - variants.append(_symbols.TypeReference(unknown, self.module)) + variants.append(self._public_reference(unknown)) writer.line("@", _DATACLASS) - writer.line("class ", unknown, "(_SerdeShape):") + writer.line("class ", unknown, "(", self.serde_shape, "):") with writer.indent(): _write_docstring(writer, _UNKNOWN_UNION_DOC) writer.line() @@ -314,7 +382,7 @@ def _write_union(self, shape: _model.Shape) -> None: "def deserialize(self, deserializer: ", _SHAPE_DESERIALIZER, ") -> ", - _symbols.TypeReference(name, self.module), + self._public_reference(name), ": ...", ) @@ -341,12 +409,44 @@ def _write_enum(self, shape: _model.Shape) -> None: if not docs and not shape.members: writer.line("pass") + def _type_reference(self, shape_id: _model.ShapeId) -> _symbols.TypeReference: + return self._partition_reference(self.symbols.type_reference(shape_id)) + + def _public_reference(self, name: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, "..models" if self.external_bases else self.module + ) + + def _partition_reference( + self, reference: _symbols.TypeReference + ) -> _symbols.TypeReference: + arguments = tuple(map(self._partition_reference, reference.arguments)) + module = ( + "..models" + if self.external_bases and reference.module == self.module + else reference.module + ) + if module == reference.module and arguments == reference.arguments: + return reference + return replace(reference, module=module, arguments=arguments) + def _model_exports(plan: ModelPlan) -> tuple[str, ...]: exports = ("ServiceError", *plan.registry_index) return (*exports, "Unit") if plan.includes_unit else exports +def _stub_shape_names(plan: ModelPlan, shape: _model.Shape) -> tuple[str, ...]: + declaration = plan.declaration_names[shape.id] + if shape.type is not _model.ShapeType.UNION: + return (declaration,) + return ( + *(plan.union_variant_names[member.id] for member in shape.members.values()), + plan.union_unknown_names[shape.id], + declaration, + ) + + def _write_mapping(writer: PythonWriter, name: str, values: Mapping[str, str]) -> None: if not values: writer.line(name, " = {}") diff --git a/packages/smithy-python/src/smithy_python/schemas.py b/packages/smithy-python/src/smithy_python/schemas.py index c47084e45..cf0764641 100644 --- a/packages/smithy-python/src/smithy_python/schemas.py +++ b/packages/smithy-python/src/smithy_python/schemas.py @@ -27,6 +27,7 @@ class SchemaPlan: """Derived semantic data used to emit compact schemas.""" shapes: tuple[_model.Shape, ...] + partition_groups: tuple[tuple[_model.Shape, ...], ...] names: Mapping[_model.ShapeId, str] strings: Mapping[str, str] traits: Mapping[_model.ShapeId, _TraitPlan] @@ -39,7 +40,7 @@ def from_shapes( symbols: _symbols.SymbolProvider, ) -> SchemaPlan: """Plan schema order, filtered traits, names, and interned strings.""" - emitted = _schema_order(shapes) + emitted, partition_groups = _schema_order(shapes) included = _included_traits(model) inputs = { operation.input for operation in model.iter_shapes(_model.OperationShape) @@ -69,6 +70,7 @@ def from_shapes( return cls( shapes=emitted, + partition_groups=partition_groups, names=MappingProxyType( {shape.id: symbols.schema_name(shape.id) for shape in emitted} ), @@ -83,18 +85,111 @@ class SchemaGenerator: def generate(self, context: GenerationContext[TypesSettings]) -> None: """Register ``._private.schemas`` with the file manifest.""" package = context.settings.package - path = f"src/{package.replace('.', '/')}/_private/schemas.py" - writer = context.files.python_module(path, f"{package}._private.schemas") + directory = package.replace(".", "/") plan = context.plan.schemas - for value, name in plan.strings.items(): + size = context.settings.schema_partition_size + if size is None or len(plan.shapes) <= size: + writer = context.files.python_module( + f"src/{directory}/_private/schemas.py", + f"{package}._private.schemas", + ) + _write_schema_module(writer, plan, plan.shapes) + return + + partitions = _schema_partitions(plan, size) + _write_schema_facade( + context.files.python_module( + f"src/{directory}/_private/schemas.py", + f"{package}._private.schemas", + ), + plan, + partitions, + ) + for index, shapes in enumerate(partitions): + _write_schema_module( + context.files.python_module( + f"src/{directory}/_private/schemas_{index}.py", + f"{package}._private.schemas_{index}", + ), + plan, + shapes, + facade="schemas", + ) + + +_IMPORT_MODULE: Final = _symbols.TypeReference("import_module", "importlib") + + +def _write_schema_facade( + writer: PythonWriter, + plan: SchemaPlan, + partitions: tuple[tuple[_model.Shape, ...], ...], +) -> None: + writer.line("_PARTITION_NAMES: tuple[tuple[str, ...], ...] = (") + with writer.indent(): + for shapes in partitions: + writer.line("(") + with writer.indent(): + for shape in shapes: + writer.line(repr(plan.names[shape.id]), ",") + writer.line("),") + writer.line(")") + writer.line("_PARTITIONS = {") + with writer.indent(): + writer.line("name: index") + writer.line("for index, names in enumerate(_PARTITION_NAMES)") + writer.line("for name in names") + writer.line("}") + writer.line("del _PARTITION_NAMES") + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("try:") + with writer.indent(): + writer.line("partition = _PARTITIONS[name]") + writer.line("except KeyError as error:") + with writer.indent(): + writer.line("raise AttributeError(name) from error") + writer.line( + "module = ", + _IMPORT_MODULE, + '(f".schemas_{partition}", __package__)', + ) + writer.line("value = getattr(module, name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *_PARTITIONS})") + + +def _write_schema_module( + writer: PythonWriter, + plan: SchemaPlan, + shapes: tuple[_model.Shape, ...], + *, + facade: str | None = None, +) -> None: + local = frozenset(shape.id for shape in shapes) + if facade is not None and any( + member.target in plan.names and member.target not in local + for shape in shapes + for member in shape.members.values() + ): + writer.line("from . import ", facade) + writer.line() + strings = _schema_strings(plan, shapes) + for value, name in plan.strings.items(): + if value in strings: writer.line(name, " = ", repr(value)) - if plan.strings: - writer.line() + if strings: + writer.line() - written: set[_model.ShapeId] = set() - for shape in plan.shapes: - _write_shape(writer, plan, shape, written) - written.add(shape.id) + written: set[_model.ShapeId] = set() + for shape in shapes: + _write_shape(writer, plan, shape, written, local=local, facade=facade) + written.add(shape.id) # This contract is intentionally local to the generator. It must match @@ -264,6 +359,9 @@ def _write_shape( plan: SchemaPlan, shape: _model.Shape, written: set[_model.ShapeId], + *, + local: Collection[_model.ShapeId], + facade: str | None, ) -> None: writer.line(plan.names[shape.id], ": ", _STATIC_SCHEMA, " = (") with writer.indent(): @@ -284,7 +382,13 @@ def _write_shape( for member in shape.members.values(): writer.line( "(", - _schema_ref(plan, member.target, written), + *_schema_ref( + plan, + member.target, + written, + local=local, + facade=facade, + ), ", ", *_trait_parts(plan, plan.traits[member.id]), "),", @@ -406,23 +510,36 @@ def _node_parts( def _schema_ref( - plan: SchemaPlan, target: _model.ShapeId, written: set[_model.ShapeId] -) -> str: + plan: SchemaPlan, + target: _model.ShapeId, + written: set[_model.ShapeId], + *, + local: Collection[_model.ShapeId], + facade: str | None, +) -> tuple[_SourcePart, ...]: if (prelude := _PRELUDE_INTS.get(target)) is not None: - return str(prelude) + return (str(prelude),) try: name = plan.names[target] except KeyError as error: raise _exceptions.CodegenError( f"Schema target {target} was not selected" ) from error - return name if target in written else f"lambda: {name}" + if target in local: + return (name if target in written else f"lambda: {name}",) + if facade is None: + raise _exceptions.CodegenError(f"Schema target {target} is not in this module") + return ("lambda: ", facade, ".", name) def _schema_order( shapes: tuple[_model.Shape, ...], -) -> tuple[_model.Shape, ...]: +) -> tuple[ + tuple[_model.Shape, ...], + tuple[tuple[_model.Shape, ...], ...], +]: by_id = {shape.id: shape for shape in shapes} + positions = {shape.id: index for index, shape in enumerate(shapes)} graph = { shape.id: tuple( member.target for member in shape.members.values() if member.target in by_id @@ -457,6 +574,7 @@ def _schema_order( pending.append((target, 0)) recursive: set[_model.ShapeId] = set() + recursive_components: dict[_model.ShapeId, tuple[_model.ShapeId, ...]] = {} visited.clear() for root in reversed(finished): if root in visited: @@ -473,11 +591,97 @@ def _schema_order( component_pending.append(target) if len(component) > 1 or root in graph[root]: recursive.update(component) + ordered_component = tuple(sorted(component, key=positions.__getitem__)) + for shape_id in component: + recursive_components[shape_id] = ordered_component - return ( + emitted = ( *(by_id[shape_id] for shape_id in finished if shape_id not in recursive), *(shape for shape in shapes if shape.id in recursive), ) + groups: list[tuple[_model.Shape, ...]] = [] + grouped: set[_model.ShapeId] = set() + for shape in emitted: + if shape.id in grouped: + continue + group_ids = recursive_components.get(shape.id, (shape.id,)) + groups.append(tuple(by_id[shape_id] for shape_id in group_ids)) + grouped.update(group_ids) + return emitted, tuple(groups) + + +def _schema_partitions( + plan: SchemaPlan, + size: int, +) -> tuple[tuple[_model.Shape, ...], ...]: + groups = plan.partition_groups + partitions: list[tuple[_model.Shape, ...]] = [] + current: list[_model.Shape] = [] + for group in groups: + if current and len(current) + len(group) > size: + partitions.append(tuple(current)) + current.clear() + current.extend(group) + if current: + partitions.append(tuple(current)) + partition_count = len(partitions) + if partition_count < 2: + return tuple(partitions) + + weights = tuple(_schema_group_weight(plan, group) for group in groups) + partitions.clear() + start = 0 + remaining_weight = sum(weights) + for remaining_parts in range(partition_count, 1, -1): + target = remaining_weight / remaining_parts + end = start + weight = 0 + last_end = len(groups) - remaining_parts + 1 + while end < last_end: + next_weight = weight + weights[end] + if weight and abs(target - weight) <= abs(target - next_weight): + break + weight = next_weight + end += 1 + partitions.append( + tuple(shape for group in groups[start:end] for shape in group) + ) + start = end + remaining_weight -= weight + partitions.append(tuple(shape for group in groups[start:] for shape in group)) + return tuple(partitions) + + +def _schema_group_weight(plan: SchemaPlan, group: tuple[_model.Shape, ...]) -> int: + declarations = sum( + len(plan.strings[value]) + len(repr(value)) + 4 + for value in _schema_strings(plan, group) + ) + return declarations + sum(_schema_shape_weight(plan, shape) for shape in group) + + +def _schema_shape_weight(plan: SchemaPlan, shape: _model.Shape) -> int: + """Estimate compact-schema source bytes without rendering shapes twice.""" + return ( + 60 + + len(shape.id.namespace) + + 2 * len(shape.id.name) + + _source_size(_trait_parts(plan, plan.traits[shape.id])) + + sum( + 40 + + 2 * len(member.name) + + len(plan.names.get(member.target, member.target.name)) + + _source_size(_trait_parts(plan, plan.traits[member.id])) + for member in shape.members.values() + ) + ) + + +def _source_size(parts: Collection[_SourcePart]) -> int: + return sum( + len(part.name if isinstance(part, _symbols.TypeReference) else part) + for part in parts + ) def _intern_traits(strings: _StringStore, traits: _TraitPlan) -> None: @@ -489,6 +693,28 @@ def _intern_traits(strings: _StringStore, traits: _TraitPlan) -> None: strings.intern(value) +def _schema_strings( + plan: SchemaPlan, shapes: Collection[_model.Shape] +) -> frozenset[str]: + values: set[str] = set() + for shape in shapes: + values.update((shape.id.namespace, shape.id.name)) + _collect_trait_strings(values, plan.traits[shape.id]) + for member in shape.members.values(): + values.add(member.name) + _collect_trait_strings(values, plan.traits[member.id]) + return frozenset(values) + + +def _collect_trait_strings(values: set[str], traits: _TraitPlan) -> None: + if isinstance(traits, int): + return + for trait_id, value in traits.items(): + values.add(str(trait_id)) + if isinstance(value, str): + values.add(value) + + class _StringStore: __slots__ = ("_allocated", "_next", "values") diff --git a/packages/smithy-python/tests/unit/test_generation.py b/packages/smithy-python/tests/unit/test_generation.py index 7be37ca29..6065e8b4f 100644 --- a/packages/smithy-python/tests/unit/test_generation.py +++ b/packages/smithy-python/tests/unit/test_generation.py @@ -33,6 +33,15 @@ def test_types_settings_default_to_identity_selection() -> None: ) assert settings.selector == Selector.identity() assert not settings.generate_inputs_and_outputs + assert settings.schema_partition_size == 250 + assert settings.stub_partition_size == 250 + + +def test_types_settings_require_positive_partition_size() -> None: + with pytest.raises(ValueError, match="Partition sizes must be positive"): + TypesSettings("example", "1.0.0", None, schema_partition_size=0) + with pytest.raises(ValueError, match="Partition sizes must be positive"): + TypesSettings("example", "1.0.0", None, stub_partition_size=0) def test_generation_context_shares_manifest_dependencies() -> None: diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py index 2d93ae8d6..db80f53b1 100644 --- a/packages/smithy-python/tests/unit/test_models.py +++ b/packages/smithy-python/tests/unit/test_models.py @@ -44,6 +44,7 @@ def _context( *, service_id: str | None = "example#Service", package: str = _PACKAGE, + stub_partition_size: int | None = 250, ) -> GenerationContext[TypesSettings]: model = load_model(model_data) selection = select_type_shapes( @@ -51,7 +52,12 @@ def _context( service_id=service_id, generate_inputs_and_outputs=True, ) - settings = TypesSettings(package, "1.0.0", None) + settings = TypesSettings( + package, + "1.0.0", + None, + stub_partition_size=stub_partition_size, + ) symbols = SymbolProvider( model, selection, @@ -191,8 +197,15 @@ def _model() -> dict[str, object]: } -def _generate(tmp_path: Path) -> tuple[GenerationContext[TypesSettings], Path]: - context = _context(_model()) +def _generate( + tmp_path: Path, + *, + stub_partition_size: int | None = 250, +) -> tuple[GenerationContext[TypesSettings], Path]: + context = _context( + _model(), + stub_partition_size=stub_partition_size, + ) SchemaGenerator().generate(context) ModelGenerator().generate(context) context.files.text_file(f"src/{_PACKAGE}/__init__.py", "") @@ -433,6 +446,68 @@ def read_struct(self, schema: Any, consumer: Any) -> None: models.Original +def test_partitioned_stub_preserves_public_models( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate( + tmp_path, + stub_partition_size=2, + ) + package = source_root / _PACKAGE + facade = (package / "models.pyi").read_text() + partitions = sorted((package / "_private").glob("models_*.pyi")) + assert len(partitions) == 4 + assert "class Input(" not in facade + assert "from ._private.models_0 import (" in facade + assert "Input," in facade + assert "Input as Input" not in facade + assert "_SelectionDeserializer" not in facade + assert any("class Input(" in path.read_text() for path in partitions) + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + assert models.Renamed(value="value").value == "value" + assert models.SelectionText(value="value").value == "value" + + generated = tuple( + str(path) + for path in sorted(package.rglob("*")) + if path.suffix in {".py", ".pyi"} + ) + result = _run_tool("ruff", "check", "--isolated", *generated) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize(("declaration_count", "partition_count"), [(250, 0), (251, 2)]) +def test_stub_partition_threshold( + tmp_path: Path, + declaration_count: int, + partition_count: int, +) -> None: + package = "generated_threshold" + context = _context( + { + "smithy": "2.0", + "shapes": { + f"example#Shape{index}": {"type": "structure"} + for index in range(declaration_count) + }, + }, + service_id=None, + package=package, + ) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + source = tmp_path / f"src/{package}" + assert len(tuple((source / "_private").glob("models_*.pyi"))) == partition_count + facade = (source / "models.pyi").read_text() + assert ("._private.models_0" in facade) is bool(partition_count) + + def test_generated_model_names_do_not_shadow_runtime_helpers( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -750,10 +825,12 @@ def test_modeled_unit_does_not_shadow_prelude_unit( assert isinstance(decoded.action.value, models.Unit) +@pytest.mark.parametrize("stub_partition_size", [None, 2]) def test_generated_stub_passes_strict_pyright( tmp_path: Path, + stub_partition_size: int | None, ) -> None: - _, source_root = _generate(tmp_path) + _, source_root = _generate(tmp_path, stub_partition_size=stub_partition_size) (tmp_path / "check.py").write_text( ( f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" diff --git a/packages/smithy-python/tests/unit/test_schemas.py b/packages/smithy-python/tests/unit/test_schemas.py index ca5bc72cb..3a8b81f06 100644 --- a/packages/smithy-python/tests/unit/test_schemas.py +++ b/packages/smithy-python/tests/unit/test_schemas.py @@ -4,7 +4,9 @@ from __future__ import annotations +import importlib import runpy +import sys from collections.abc import Mapping from pathlib import Path from types import MappingProxyType @@ -32,10 +34,19 @@ from smithy_python.symbols import SymbolProvider -def _context(model_data: Mapping[str, object]) -> GenerationContext[TypesSettings]: +def _context( + model_data: Mapping[str, object], + *, + schema_partition_size: int | None = 250, +) -> GenerationContext[TypesSettings]: model = load_model(model_data) selection = select_type_shapes(model, generate_inputs_and_outputs=True) - settings = TypesSettings("example.types", "1.0.0", None) + settings = TypesSettings( + "example.types", + "1.0.0", + None, + schema_partition_size=schema_partition_size, + ) symbols = SymbolProvider(model, selection, package=settings.package) dependencies = DependencySet() return GenerationContext( @@ -231,6 +242,158 @@ def test_schema_generator_emits_lazy_hydratable_schemas(tmp_path: Path) -> None: assert str(node.members_by_index[2].expect_member_target().id) == "example#Choice" +def test_schema_generator_partitions_behind_lazy_facade( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Leaf": {"type": "string"}, + "example#Node": { + "type": "structure", + "members": {"leaf": {"target": "example#Leaf"}}, + }, + "example#CycleA": { + "type": "structure", + "members": {"b": {"target": "example#CycleB"}}, + }, + "example#CycleB": { + "type": "structure", + "members": {"a": {"target": "example#CycleA"}}, + }, + }, + }, + schema_partition_size=1, + ) + SchemaGenerator().generate(context) + context.files.text_file("src/example/types/__init__.py", "") + context.files.text_file("src/example/types/_private/__init__.py", "") + context.files.commit(tmp_path) + + package = tmp_path / "src/example/types/_private" + assert (package / "schemas.py").is_file() + assert (package / "schemas_0.py").is_file() + assert (package / "schemas_1.py").is_file() + facade_source = (package / "schemas.py").read_text() + assert "_PARTITION_NAMES: tuple[tuple[str, ...], ...]" in facade_source + assert "name: index" in facade_source + assert facade_source.count(".schemas_") == 1 + partitions = tuple(package.glob("schemas_*.py")) + cycle = next( + path.read_text() for path in partitions if "CYCLE_A:" in path.read_text() + ) + assert "CYCLE_B:" in cycle + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(tmp_path / "src") + ) + facade = importlib.import_module("example.types._private.schemas") + assert "example.types._private.schemas_0" not in sys.modules + assert "example.types._private.schemas_1" not in sys.modules + + static_node = cast(tuple[Any, ...], facade.NODE) + assert facade.NODE is static_node + assert "NODE" in dir(facade) + assert "example.types._private.schemas_1" in sys.modules + assert "example.types._private.schemas_0" not in sys.modules + with pytest.raises(AttributeError): + facade.MISSING + node = _schema_compact.hydrate(static_node) + assert str(node.members["leaf"].expect_member_target().id) == "example#Leaf" + assert "example.types._private.schemas_0" in sys.modules + cycle_a = _schema_compact.hydrate(cast(tuple[Any, ...], facade.CYCLE_A)) + cycle_b = cycle_a.members["b"].expect_member_target() + assert cycle_b.members["a"].expect_member_target() is cycle_a + + +@pytest.mark.parametrize(("shape_count", "partition_count"), [(250, 0), (251, 2)]) +def test_schema_partition_threshold( + tmp_path: Path, + shape_count: int, + partition_count: int, +) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + f"example#Shape{index}": {"type": "structure"} + for index in range(shape_count) + }, + }, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + + private = tmp_path / "src/example/types/_private" + assert len(tuple(private.glob("schemas_*.py"))) == partition_count + facade = (private / "schemas.py").read_text() + assert ("_PARTITIONS" in facade) is bool(partition_count) + + +def test_schema_partitions_balance_emitted_size(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#AHeavy": { + "type": "structure", + "members": { + f"member{index}": {"target": "smithy.api#String"} + for index in range(20) + }, + }, + "example#BTiny": {"type": "structure"}, + "example#CTiny": {"type": "structure"}, + "example#DTiny": {"type": "structure"}, + }, + }, + schema_partition_size=2, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + private = tmp_path / "src/example/types/_private" + + first = (private / "schemas_0.py").read_text() + second = (private / "schemas_1.py").read_text() + assert "A_HEAVY:" in first + assert "B_TINY:" not in first + assert all(f"{name}:" in second for name in ("B_TINY", "C_TINY", "D_TINY")) + + +def test_schema_partition_weights_include_interned_strings(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#AHeavy": { + "type": "structure", + "members": { + "value": { + "target": "smithy.api#String", + "traits": {"smithy.api#default": "x" * 10_000}, + } + }, + }, + "example#BTiny": {"type": "structure"}, + "example#CTiny": {"type": "structure"}, + "example#DTiny": {"type": "structure"}, + }, + }, + schema_partition_size=2, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + private = tmp_path / "src/example/types/_private" + + first = (private / "schemas_0.py").read_text() + second = (private / "schemas_1.py").read_text() + assert "A_HEAVY:" in first + assert "B_TINY:" not in first + assert all(f"{name}:" in second for name in ("B_TINY", "C_TINY", "D_TINY")) + + def test_schema_generator_preserves_primitive_prelude_targets(tmp_path: Path) -> None: context = _context( {