Skip to content

Commit ebb2ad4

Browse files
kuhekuhe
committed
Internalize client generation
1 parent 76a67c6 commit ebb2ad4

29 files changed

Lines changed: 729 additions & 64 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# macOS system files
22
.DS_Store
33

4+
# IDE
5+
.idea/
6+
47
# Docs build artifacts
58
site/
69
docs/clients/

‎Makefile‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ PYTHON_VERSION := 3.12
44
# Colocated smithy-python checkout that provides the codegen jars.
55
SMITHY_PYTHON ?= ../smithy-python
66

7-
.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen publish-codegen aio-check
7+
# Clients per gradle build in `make clients`; lower it if the build runs out of heap.
8+
BATCH_SIZE ?= 25
9+
10+
.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen clients publish-codegen aio-check
811

912
# Publishes the smithy-python codegen jars to the local Maven cache so `gen`
1013
# can resolve them. Run once, and again whenever the generator changes.
@@ -14,7 +17,12 @@ publish-codegen:
1417
# Regenerate one client from its service model: `make gen dynamodb`.
1518
# Accepts dynamodb, aws_sdk_dynamodb, aws-sdk-dynamodb, or a hyphenated model name.
1619
gen:
17-
SMITHY_PYTHON=$(SMITHY_PYTHON) uv run python codegen/gen_client.py $(filter-out gen,$(MAKECMDGOALS))
20+
SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py $(filter-out gen,$(MAKECMDGOALS))
21+
22+
# Regenerate every client from codegen/aws-models. Each client is cleared first,
23+
# keeping only the paths in gen_client.py's KEEP, then scripts/client-overlays is applied.
24+
clients:
25+
SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py --all --batch-size $(BATCH_SIZE)
1826

1927
# Swallow the client-name argument so make does not treat it as a target.
2028
# Scoped to `gen` so it does not mask typos in unrelated commands.

‎codegen/gen_client.py‎

Lines changed: 215 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,125 +1,278 @@
11
#!/usr/bin/env python3
2-
"""Generate one aws-sdk-python client from its service model via smithy-python codegen.
2+
"""Regenerate aws-sdk-python clients from service models via smithy-python codegen.
33
4-
Usage: python codegen/gen_client.py <name>
4+
Usage: python codegen/gen_client.py (--all | <name>...)
55
<name> accepts dynamodb, aws_sdk_dynamodb, aws-sdk-dynamodb, or bedrock-runtime forms.
66
"""
77

8+
import argparse
89
import json
10+
import os
11+
import re
912
import shutil
1013
import subprocess
1114
import sys
15+
import tomllib
1216
from pathlib import Path
1317

1418
CODEGEN_DIR = Path(__file__).resolve().parent
1519
REPO = CODEGEN_DIR.parent
1620
MODELS = CODEGEN_DIR / "aws-models"
1721
CLIENTS = REPO / "clients"
22+
OVERLAYS = REPO / "scripts" / "client-overlays"
1823
PROJECT = "aws-sdk-python-codegen"
24+
META_PACKAGE = "aws-sdk-python"
25+
DEFAULT_BATCH_SIZE = 25
26+
27+
# Paths codegen never emits; they survive the pre-generation delete. A path codegen
28+
# does emit but that must differ from codegen output belongs in client-overlays.
29+
KEEP = (
30+
".changes",
31+
"CHANGELOG.md",
32+
"LICENSE",
33+
"NOTICE",
34+
"Makefile",
35+
"docs",
36+
"examples",
37+
"scripts",
38+
"tests/integration",
39+
"zensical.toml",
40+
)
41+
42+
# The integration tests of these clients set up resources with boto3.
43+
CLIENTS_REQUIRING_BOTO3 = {
44+
"aws-sdk-connecthealth",
45+
"aws-sdk-lex-runtime-v2",
46+
"aws-sdk-transcribe-streaming",
47+
}
48+
CLIENT_DOC_DEPS = {"zensical", "mkdocstrings[python]"}
49+
50+
51+
def main() -> None:
52+
parser = argparse.ArgumentParser()
53+
group = parser.add_mutually_exclusive_group(required=True)
54+
group.add_argument("--all", action="store_true")
55+
group.add_argument("names", nargs="*", default=[])
56+
parser.add_argument("-b", "--batch-size", type=int, default=DEFAULT_BATCH_SIZE)
57+
args = parser.parse_args()
58+
if args.batch_size < 1:
59+
sys.exit("--batch-size must be at least 1")
60+
61+
models = (
62+
sorted(MODELS.glob("*.json"))
63+
if args.all
64+
else [resolve_model(n) for n in args.names]
65+
)
66+
if not models:
67+
sys.exit("no clients selected")
68+
69+
docs_deps = _docs_deps()
70+
for start in range(0, len(models), args.batch_size):
71+
batch = models[start : start + args.batch_size]
72+
_build(batch)
73+
for model in batch:
74+
_install_client(model.stem, docs_deps)
1975

2076

2177
def resolve_model(name: str) -> Path:
22-
"""Map any client-name form to its aws-models/<model>.json file."""
2378
key = name.strip()
2479
for prefix in ("aws-sdk-", "aws_sdk_"):
2580
if key.startswith(prefix):
26-
key = key[len(prefix):]
81+
key = key[len(prefix) :]
2782
break
28-
# Models are hyphenated on disk; accept underscores too.
29-
candidates = {key, key.replace("_", "-"), key.replace("-", "_")}
30-
for cand in candidates:
83+
for cand in {key, key.replace("_", "-"), key.replace("-", "_")}:
3184
model = MODELS / f"{cand}.json"
3285
if model.exists():
3386
return model
3487
available = sorted(p.stem for p in MODELS.glob("*.json"))
35-
sys.exit(
36-
f"no model for '{name}' in {MODELS}\navailable: {', '.join(available)}"
37-
)
88+
sys.exit(f"no model for '{name}' in {MODELS}\navailable: {', '.join(available)}")
3889

3990

4091
def service_id(model: Path) -> str:
4192
shapes = json.loads(model.read_text())["shapes"]
4293
services = [k for k, v in shapes.items() if v.get("type") == "service"]
4394
if len(services) != 1:
44-
sys.exit(f"expected exactly one service shape in {model.name}, found {services}")
95+
sys.exit(
96+
f"expected exactly one service shape in {model.name}, found {services}"
97+
)
4598
return services[0]
4699

47100

48-
def main() -> None:
49-
if len(sys.argv) != 2:
50-
sys.exit("usage: gen_client.py <client-name>")
51-
52-
model = resolve_model(sys.argv[1])
53-
slug = model.stem # e.g. bedrock-runtime
54-
module = "aws_sdk_" + slug.replace("-", "_") # aws_sdk_bedrock_runtime
55-
client_dir_name = "aws-sdk-" + slug # aws-sdk-bedrock-runtime
56-
svc = service_id(model)
57-
58-
version = "0.0.1"
59-
client_pyproject = CLIENTS / client_dir_name / "pyproject.toml"
60-
if client_pyproject.exists():
61-
for line in client_pyproject.read_text().splitlines():
62-
if line.startswith("version"):
63-
version = line.split("=", 1)[1].strip().strip('"')
64-
break
101+
def _projection(model: Path) -> dict[str, object]:
102+
slug = model.stem
103+
# Per-projection imports make each client's input independent of its batch.
104+
return {
105+
"imports": [str(model)],
106+
"plugins": {
107+
"python-client-codegen": {
108+
"service": service_id(model),
109+
"module": _module(slug),
110+
"moduleVersion": _client_version(slug),
111+
}
112+
}
113+
}
114+
115+
116+
def _module(slug: str) -> str:
117+
return "aws_sdk_" + slug.replace("-", "_")
118+
119+
120+
def _client_version(slug: str) -> str:
121+
pyproject = CLIENTS / f"aws-sdk-{slug}" / "pyproject.toml"
122+
if pyproject.exists():
123+
return _read_version(pyproject)
124+
# As in the release build: clients share major.minor, so a new one starts at .0.
125+
for existing in sorted(CLIENTS.glob("aws-sdk-*/pyproject.toml")):
126+
if existing.parent.name != META_PACKAGE:
127+
major, minor, _ = _read_version(existing).split(".")
128+
return f"{major}.{minor}.0"
129+
sys.exit("cannot determine a version for new client: no existing client has one")
65130

131+
132+
def _read_version(pyproject: Path) -> str:
133+
with pyproject.open("rb") as f:
134+
return tomllib.load(f)["project"]["version"]
135+
136+
137+
def _build(models: list[Path]) -> None:
66138
build_config = {
67139
"version": "1.0",
68-
"imports": [str(model)],
69-
"projections": {
70-
slug: {
71-
"transforms": [
72-
{"name": "includeServices", "args": {"services": [svc]}},
73-
{"name": "removeUnusedShapes"},
74-
],
75-
"plugins": {
76-
"python-client-codegen": {
77-
"service": svc,
78-
"module": module,
79-
"moduleVersion": version,
80-
}
81-
},
82-
}
83-
},
140+
"projections": {m.stem: _projection(m) for m in models},
84141
}
85142
build_file = CODEGEN_DIR / "smithy-build.json"
86143
build_file.write_text(json.dumps(build_config, indent=4) + "\n")
87-
print(f"generating {client_dir_name} ({svc}) -> module {module} v{version}")
88-
144+
print(f"generating {len(models)} client(s): {', '.join(m.stem for m in models)}")
89145
subprocess.run(
90146
[str(CODEGEN_DIR / "gradlew"), "-p", str(CODEGEN_DIR), "clean", "build", "-q"],
91147
check=True,
92148
)
93149

150+
151+
def _install_client(slug: str, docs_deps: list[str]) -> None:
152+
name = f"aws-sdk-{slug}"
94153
projection = (
95-
CODEGEN_DIR / "build" / "smithyprojections" / PROJECT / slug / "python-client-codegen"
154+
CODEGEN_DIR
155+
/ "build"
156+
/ "smithyprojections"
157+
/ PROJECT
158+
/ slug
159+
/ "python-client-codegen"
96160
)
97161
if not projection.is_dir():
98162
sys.exit(f"codegen produced no output at {projection}")
163+
generated = _relative_files(projection)
164+
_check_keep_untouched(name, generated)
165+
overlays = _overlay_files(name, generated)
99166

100-
dest_src = CLIENTS / client_dir_name / "src" / module
101-
if dest_src.exists():
102-
shutil.rmtree(dest_src)
103-
dest_src.parent.mkdir(parents=True, exist_ok=True)
104-
shutil.copytree(projection / "src" / module, dest_src)
167+
dest = CLIENTS / name
168+
_clear(dest)
169+
shutil.copytree(projection, dest, dirs_exist_ok=True)
170+
for rel in overlays:
171+
shutil.copyfile(OVERLAYS / name / rel, dest / rel)
105172

106-
_format(dest_src, CLIENTS / client_dir_name / "pyproject.toml")
107-
print(f"wrote {dest_src}")
173+
if docs_deps:
174+
_uv_add(dest, "docs", docs_deps)
175+
if name in CLIENTS_REQUIRING_BOTO3:
176+
_uv_add(dest, "test", ["boto3"])
177+
_format(dest / "src" / _module(slug), dest / "pyproject.toml")
178+
print(f"wrote {dest}")
108179

109180

110-
def _format(src_dir: Path, config: Path) -> None:
111-
"""Format generated code with smithy-python's lockfile-pinned ruff.
181+
def _relative_files(root: Path) -> set[str]:
182+
return {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()}
183+
184+
185+
def _is_kept(rel: str) -> bool:
186+
return any(rel == k or rel.startswith(k + "/") for k in KEEP)
187+
188+
189+
def _check_keep_untouched(name: str, generated: set[str]) -> None:
190+
clashes = sorted(rel for rel in generated if _is_kept(rel))
191+
if clashes:
192+
sys.exit(
193+
f"{name}: codegen now emits kept paths {clashes}; "
194+
"remove them from KEEP and move any hand edits to scripts/client-overlays"
195+
)
196+
197+
198+
def _overlay_files(name: str, generated: set[str]) -> list[str]:
199+
root = OVERLAYS / name
200+
if not root.is_dir():
201+
return []
202+
overlays = sorted(_relative_files(root))
203+
stray = [rel for rel in overlays if rel not in generated]
204+
if stray:
205+
sys.exit(
206+
f"{name}: overlays {stray} do not replace any generated file; "
207+
"move them into the client and add them to KEEP"
208+
)
209+
return overlays
210+
211+
212+
def _clear(dest: Path) -> None:
213+
if not dest.exists():
214+
return
215+
ignored = _ignored(dest)
216+
for path in sorted(dest.rglob("*"), key=lambda p: len(p.parts), reverse=True):
217+
rel = path.relative_to(dest).as_posix()
218+
if (
219+
_is_kept(rel)
220+
or rel in ignored
221+
or any(rel.startswith(i + "/") for i in ignored)
222+
):
223+
continue
224+
if path.is_dir() and not path.is_symlink():
225+
if not any(path.iterdir()):
226+
path.rmdir()
227+
else:
228+
path.unlink()
229+
230+
231+
def _ignored(dest: Path) -> set[str]:
232+
# Gitignored files (venvs, lockfiles) are local state, not leftover codegen output.
233+
out = subprocess.run(
234+
[
235+
"git",
236+
"ls-files",
237+
"--others",
238+
"--ignored",
239+
"--exclude-standard",
240+
"--directory",
241+
"-z",
242+
".",
243+
],
244+
cwd=dest,
245+
check=True,
246+
capture_output=True,
247+
text=True,
248+
).stdout
249+
return {p.rstrip("/") for p in out.split("\0") if p}
250+
251+
252+
def _docs_deps() -> list[str]:
253+
deps = []
254+
for line in (REPO / "requirements-docs.in").read_text().splitlines():
255+
line = line.split("#")[0].strip()
256+
if line and re.split(r"[<>=!~]", line)[0] in CLIENT_DOC_DEPS:
257+
deps.append(line)
258+
return deps
112259

113-
The generator only formats when ruff is on its own PATH, which gradle's env
114-
lacks; run the pinned formatter explicitly so output matches committed style.
115-
"""
116-
import os
117260

118-
smithy_python = Path(os.environ.get("SMITHY_PYTHON", REPO.parent / "smithy-python"))
261+
def _uv_add(client: Path, group: str, deps: list[str]) -> None:
262+
subprocess.run(
263+
["uv", "add", "--group", group, "--frozen", *deps], cwd=client, check=True
264+
)
265+
266+
267+
def _format(src_dir: Path, config: Path) -> None:
268+
# The generator only formats when ruff is on gradle's PATH, which it usually is not.
269+
smithy_python = Path(
270+
os.environ.get("SMITHY_PYTHON", REPO.parent / "smithy-python")
271+
).resolve()
119272
if not (smithy_python / "pyproject.toml").exists():
120273
print(f"skipping ruff: no smithy-python checkout at {smithy_python}")
121274
return
122-
cfg = ["--config", str(config)] if config.exists() else []
275+
cfg = ["--config", str(config)]
123276
for cmd in (
124277
["uv", "run", "ruff", "check", str(src_dir), "--fix", *cfg],
125278
["uv", "run", "ruff", "format", str(src_dir), *cfg],

‎codegen/gradle.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.gradle.jvmargs=-Xmx4096M

‎scripts/client-overlays/aws-sdk-api-gateway/README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Amazon API Gateway Client
2+
3+
The `aws-sdk-api-gateway` package provides a client for making
4+
requests to Amazon API Gateway.
5+
6+
> **Note:** This SDK is experimental and in early development. Breaking changes
7+
> may occur prior to the release of version 1.0.0.
8+
9+
## Installation
10+
11+
```bash
12+
python -m pip install aws-sdk-api-gateway
13+
```
14+
15+
## Documentation
16+
17+
- [API Reference](https://docs.aws.amazon.com/sdk-for-python/v4/reference/clients/api-gateway/) -
18+
the API Gateway client, available operations, and types.
19+
- [Developer Guide](https://docs.aws.amazon.com/sdk-for-python/v4/guide/) -
20+
configuration, authentication, and usage of the AWS SDK for Python v4.

0 commit comments

Comments
 (0)