Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,6 @@ dist/

# scratch workspace
/workspace

# IDE
.idea
42 changes: 41 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,13 @@ PYTHON_VERSION := 3.12
# Colocated smithy-python checkout that provides the codegen jars.
SMITHY_PYTHON ?= ../smithy-python

.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen publish-codegen
.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen publish-codegen dev-install test test-integration typecheck ruff

# Each client and hand-written package is its own project with its own pytest
# and pyright config, so the check targets run them one at a time.
PROJECTS := $(patsubst %/pyproject.toml,%,$(wildcard clients/*/pyproject.toml packages/*/pyproject.toml))
SMITHY_PACKAGES := aws-sdk-signers smithy-core smithy-json smithy-cbor smithy-xml smithy-aws-core smithy-aws-event-stream smithy-test
VENV_BIN := $(CURDIR)/.venv/bin

# Publishes the smithy-python codegen jars to the local Maven cache so `gen`
# can resolve them. Run once, and again whenever the generator changes.
Expand All @@ -25,6 +31,40 @@ endif
venv:
uv venv --python $(PYTHON_VERSION)

# Installs every client and package against the colocated smithy-python
# checkout, so checks run against unreleased runtime changes.
dev-install:
@[ -d .venv ] || uv venv --python $(PYTHON_VERSION)
uv pip install --python $(VENV_BIN)/python \
$(foreach p,$(SMITHY_PACKAGES),-e $(SMITHY_PYTHON)/packages/$(p)) \
-e "$(SMITHY_PYTHON)/packages/smithy-http[aiohttp,awscrt]" \
$(foreach p,$(PROJECTS),-e ./$(p)) \
"pytest>=9.0.1,<10.0.0" "pytest-asyncio>=1.3.0,<1.4.0" \
"boto3==1.43.120" "boto3-stubs[iam,lexv2-models,s3,sts]==1.43.120" \
"pyright==1.1.406" "ruff==0.15.22"

test: dev-install
@failed=; for t in $(wildcard packages/*/tests/unit); do \
$(VENV_BIN)/python -m pytest -q $$t || failed="$$failed $$t"; \
done; [ -z "$$failed" ] || { echo "unit tests failed in:$$failed"; exit 1; }

# Requires AWS credentials for the accounts and regions the suites target.
test-integration: dev-install
@failed=; for p in $(PROJECTS); do \
[ -d $$p/tests/integration ] || continue; \
$(VENV_BIN)/python -m pytest -q $$p/tests/integration || failed="$$failed $$p"; \
done; [ -z "$$failed" ] || { echo "integration tests failed in:$$failed"; exit 1; }

typecheck: dev-install
@failed=; for p in $(PROJECTS); do \
dirs=$$(cd $$p && ls -d src tests 2>/dev/null); [ -n "$$dirs" ] || continue; \
(cd $$p && $(VENV_BIN)/pyright --pythonpath $(VENV_BIN)/python $$dirs) || failed="$$failed $$p"; \
done; [ -z "$$failed" ] || { echo "typecheck failed in:$$failed"; exit 1; }

ruff: dev-install
$(VENV_BIN)/ruff check clients packages
$(VENV_BIN)/ruff format --check clients packages

docs-lock: venv
uv pip compile requirements-docs.in -o requirements-docs.txt

Expand Down
76 changes: 55 additions & 21 deletions clients/aws-sdk-api-gateway/scripts/docs/generate_doc_stubs.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,9 @@ class ClientInfo:
class DocStubGenerator:
"""Generate markdown API Reference stubs for AWS SDK for Python v4 clients."""

def __init__(self, client_dir: Path, output_dir: Path, service_name: str | None = None) -> None:
def __init__(
self, client_dir: Path, output_dir: Path, service_name: str | None = None
) -> None:
"""
Initialize the documentation generator.

Expand Down Expand Up @@ -190,7 +192,9 @@ def _analyze_client_package(self, package_name: str) -> ClientInfo:
required = ("client", "config", "models")
missing = [name for name in required if not package.modules.get(name)]
if missing:
raise ValueError(f"Missing required modules in {package_name}: {', '.join(missing)}")
raise ValueError(
f"Missing required modules in {package_name}: {', '.join(missing)}"
)

# Parse submodules
client_module = package.modules["client"]
Expand All @@ -199,11 +203,15 @@ def _analyze_client_package(self, package_name: str) -> ClientInfo:

client_class = self._find_class_with_suffix(client_module, "Client")
if not client_class:
raise ValueError(f"No class ending with 'Client' found in {package_name}.client")
raise ValueError(
f"No class ending with 'Client' found in {package_name}.client"
)

config_class = self._find_class_with_suffix(config_module, "Config")
if not config_class:
raise ValueError(f"No class ending with 'Config' found in {package_name}.config")
raise ValueError(
f"No class ending with 'Config' found in {package_name}.config"
)

plugin_alias = config_module.members.get("Plugin")
if not plugin_alias:
Expand Down Expand Up @@ -288,7 +296,9 @@ def _analyze_operation(self, operation: Function) -> OperationInfo:

if output_type in stream_type_map:
stream_type = stream_type_map[output_type]
stream_args = self._get_subscript_elements(returns, f"'{operation.name}' stream type")
stream_args = self._get_subscript_elements(
returns, f"'{operation.name}' stream type"
)

if stream_type in (StreamType.INPUT, StreamType.DUPLEX):
event_input_type = stream_args[0].canonical_name
Expand Down Expand Up @@ -316,7 +326,9 @@ def _analyze_operation(self, operation: Function) -> OperationInfo:
def _get_expr(self, annotation: str | Expr | None, context: str) -> Expr:
"""Extract and validate an Expr from an annotation."""
if not isinstance(annotation, Expr):
raise TypeError(f"{context}: expected Expr, got {type(annotation).__name__}")
raise TypeError(
f"{context}: expected Expr, got {type(annotation).__name__}"
)
return annotation

def _get_subscript_elements(self, expr: Expr, context: str) -> list[Expr]:
Expand All @@ -330,7 +342,9 @@ def _get_subscript_elements(self, expr: Expr, context: str) -> list[Expr]:
return [el for el in slice_expr.elements if isinstance(el, Expr)]
return [slice_expr]

def _extract_models(self, models_module: Module, operations: list[OperationInfo]) -> ModelsInfo:
def _extract_models(
self, models_module: Module, operations: list[OperationInfo]
) -> ModelsInfo:
"""Extract structures, unions, enums, and errors from models module."""
structures, unions, enums, errors = [], [], [], []

Expand All @@ -356,14 +370,16 @@ def _extract_models(self, models_module: Module, operations: list[OperationInfo]

duplicates = []
for structure in structures:
if self._is_operation_io_type(structure.name, operations) or self._is_union_member(
structure.name, unions
):
if self._is_operation_io_type(
structure.name, operations
) or self._is_union_member(structure.name, unions):
duplicates.append(structure)

structures = [struct for struct in structures if struct not in duplicates]

return ModelsInfo(structures=structures, unions=unions, enums=enums, errors=errors)
return ModelsInfo(
structures=structures, unions=unions, enums=enums, errors=errors
)

def _is_union(self, member: Object | Alias) -> TypeGuard[Attribute]:
"""Check if a module member is a union type."""
Expand Down Expand Up @@ -399,7 +415,9 @@ def _extract_union_members(
for name in member_names:
if not (member_object := models_module.members.get(name)):
raise ValueError(f"Union member '{name}' not found in models module")
members.append(TypeInfo(name=member_object.name, module_path=member_object.path))
members.append(
TypeInfo(name=member_object.name, module_path=member_object.path)
)

return members

Expand All @@ -408,18 +426,22 @@ def _is_enum(self, member: Object | Alias) -> TypeGuard[Class]:
if not isinstance(member, Class):
return False
return any(
isinstance(base, ExprName) and base.name in ENUM_BASE_CLASSES for base in member.bases
isinstance(base, ExprName) and base.name in ENUM_BASE_CLASSES
for base in member.bases
)

def _is_error(self, member: Object | Alias) -> TypeGuard[Class]:
"""Check if a module member is an error."""
if not isinstance(member, Class):
return False
return any(
isinstance(base, ExprName) and base.name in ERROR_BASE_CLASSES for base in member.bases
isinstance(base, ExprName) and base.name in ERROR_BASE_CLASSES
for base in member.bases
)

def _is_operation_io_type(self, type_name: str, operations: list[OperationInfo]) -> bool:
def _is_operation_io_type(
self, type_name: str, operations: list[OperationInfo]
) -> bool:
"""Check if a type is used as operation input/output."""
return any(type_name in (op.input.name, op.output.name) for op in operations)

Expand All @@ -437,8 +459,12 @@ def _generate_client_docs(self, client_info: ClientInfo) -> bool:
self._generate_type_stubs(
client_info.models.structures, "structures", "Structure Class"
)
self._generate_type_stubs(client_info.models.errors, "errors", "Error Class")
self._generate_type_stubs(client_info.models.enums, "enums", "Enum Class", members=True)
self._generate_type_stubs(
client_info.models.errors, "errors", "Error Class"
)
self._generate_type_stubs(
client_info.models.enums, "enums", "Enum Class", members=True
)
self._generate_union_stubs(client_info.models.unions)
except OSError as e:
logger.error(f"Failed to write documentation files: {e}")
Expand Down Expand Up @@ -470,7 +496,9 @@ def _generate_index(self, client_info: ClientInfo) -> None:
]

for method in sorted(client_info.client_methods, key=lambda x: x.name):
lines.extend([*self._mkdocs_directive(method.module_path, heading_level=4), ""])
lines.extend(
[*self._mkdocs_directive(method.module_path, heading_level=4), ""]
)

# Operations section
if client_info.operations:
Expand Down Expand Up @@ -578,7 +606,9 @@ def _generate_operation_stubs(self, operations: list[OperationInfo]) -> None:

output_path = self.output_dir / "operations" / f"{op.name}.md"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(self._breadcrumb("Operations", op.name) + "\n".join(lines))
output_path.write_text(
self._breadcrumb("Operations", op.name) + "\n".join(lines)
)

logger.info(f"Wrote {len(operations)} operation file(s)")

Expand All @@ -600,7 +630,9 @@ def _generate_type_stubs(

output_path = self.output_dir / category / f"{item.name}.md"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(self._breadcrumb(category.title(), item.name) + "\n".join(lines))
output_path.write_text(
self._breadcrumb(category.title(), item.name) + "\n".join(lines)
)

logger.info(f"Wrote {len(items)} {category} file(s)")

Expand All @@ -624,7 +656,9 @@ def _generate_union_stubs(self, unions: list[UnionInfo]) -> None:

output_path = self.output_dir / "unions" / f"{union.name}.md"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(self._breadcrumb("Unions", union.name) + "\n".join(lines))
output_path.write_text(
self._breadcrumb("Unions", union.name) + "\n".join(lines)
)

logger.info(f"Wrote {len(unions)} union file(s)")

Expand Down
Loading
Loading