From 115635fb9c1212f9217ad1012efd30c8ed72a3eb Mon Sep 17 00:00:00 2001 From: Ankit Samantaray Date: Fri, 31 Jul 2026 18:09:17 +0100 Subject: [PATCH] feat(openapi): sync canonical public mirror --- .../openapi-source/build_public_mirror.py | 72 +++ .../openapi-source/bundle_canonical_source.py | 363 +++++++++++++++ .../scripts/openapi-source/source_contract.py | 429 ++++++++++++++++++ .../test_build_public_mirror.py | 29 ++ .../test_bundle_canonical_source.py | 155 +++++++ .github/workflows/sync-canonical-openapi.yml | 147 ++++++ 6 files changed, 1195 insertions(+) create mode 100644 .github/scripts/openapi-source/build_public_mirror.py create mode 100644 .github/scripts/openapi-source/bundle_canonical_source.py create mode 100644 .github/scripts/openapi-source/source_contract.py create mode 100644 .github/scripts/openapi-source/test_build_public_mirror.py create mode 100644 .github/scripts/openapi-source/test_bundle_canonical_source.py create mode 100644 .github/workflows/sync-canonical-openapi.yml diff --git a/.github/scripts/openapi-source/build_public_mirror.py b/.github/scripts/openapi-source/build_public_mirror.py new file mode 100644 index 0000000..6f7ab38 --- /dev/null +++ b/.github/scripts/openapi-source/build_public_mirror.py @@ -0,0 +1,72 @@ +"""Build the public normalized Telnyx OpenAPI mirror.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +from typing import Any + +import yaml +from bundle_canonical_source import build_bundle +from source_contract import HTTP_METHODS, canonical_semantic_bytes + + +def strip_hidden_operations(bundle: dict[str, Any]) -> tuple[dict[str, Any], int]: + public = copy.deepcopy(bundle) + removed = 0 + paths = public.get("paths") or {} + for api_path in list(paths): + path_item = paths[api_path] + if not isinstance(path_item, dict): + continue + for method in list(path_item): + operation = path_item[method] + if ( + str(method).lower() in HTTP_METHODS + and isinstance(operation, dict) + and operation.get("x-hidden") is True + ): + del path_item[method] + removed += 1 + if not any(str(key).lower() in HTTP_METHODS for key in path_item): + del paths[api_path] + return public, removed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source_root", type=Path) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--yaml-output", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + + bundle, bundle_report = build_bundle(args.source_root) + public, removed = strip_hidden_operations(bundle) + json_bytes = json.dumps(public, indent=2, ensure_ascii=True).encode() + b"\n" + yaml_text = yaml.safe_dump(public, sort_keys=False, allow_unicode=True) + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.yaml_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_bytes(json_bytes) + args.yaml_output.write_text(yaml_text, encoding="utf-8") + report = { + "schema_version": 1, + "source_file_count": bundle_report["source_file_count"], + "hidden_operation_count": removed, + "public_semantic_sha256": hashlib.sha256( + canonical_semantic_bytes(public) + ).hexdigest(), + "public_json_sha256": hashlib.sha256(json_bytes).hexdigest(), + } + args.report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/openapi-source/bundle_canonical_source.py b/.github/scripts/openapi-source/bundle_canonical_source.py new file mode 100644 index 0000000..7cc489d --- /dev/null +++ b/.github/scripts/openapi-source/bundle_canonical_source.py @@ -0,0 +1,363 @@ +"""Build one deterministic OpenAPI bundle from canonical direct-source files.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +from collections import Counter, defaultdict +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlsplit + +from source_contract import ( + SourceContractError, + assert_no_method_path_conflicts, + build_source_inventory, + canonical_semantic_bytes, + iter_source_documents, + strip_operation_code_samples, + validate_source_tree, +) + +COMPONENT_SECTIONS = ( + "schemas", + "parameters", + "securitySchemes", + "requestBodies", + "responses", + "headers", + "examples", + "links", + "callbacks", +) + + +@dataclass(frozen=True) +class ComponentAssignment: + source_file: str + section: str + original_name: str + bundled_name: str + + +def _source_prefix(source_file: str, stem_counts: Counter[str]) -> str: + path = Path(source_file) + if stem_counts[path.stem] == 1: + return path.stem + without_suffix = path.with_suffix("").as_posix() + return re.sub(r"[^A-Za-z0-9]+", "_", without_suffix).strip("_") + + +def _component_claims( + documents: Mapping[str, Mapping[str, Any]], +) -> dict[tuple[str, str], list[tuple[str, Any]]]: + claims: dict[tuple[str, str], list[tuple[str, Any]]] = defaultdict(list) + for source_file, document in documents.items(): + components = document.get("components", {}) + if not isinstance(components, dict): + raise SourceContractError(f"components must be an object: {source_file}") + unsupported = sorted(set(components) - set(COMPONENT_SECTIONS)) + if unsupported: + raise SourceContractError( + f"unsupported component sections in {source_file}: {', '.join(unsupported)}" + ) + for section in COMPONENT_SECTIONS: + entries = components.get(section, {}) + if not isinstance(entries, dict): + raise SourceContractError( + f"components/{section} must be an object: {source_file}" + ) + for name, value in entries.items(): + claims[(section, name)].append((source_file, value)) + return claims + + +def _assign_components( + documents: Mapping[str, Mapping[str, Any]], +) -> tuple[dict[tuple[str, str, str], str], dict[tuple[str, str, str], str]]: + stem_counts = Counter(Path(source).stem for source in documents) + claims_by_component = _component_claims(documents) + reserved_names = set(claims_by_component) + assignments: dict[tuple[str, str, str], str] = {} + owners: dict[tuple[str, str, str], str] = {} + used: dict[tuple[str, str], Any] = {} + + for (section, name), claims in sorted(claims_by_component.items()): + primary_source, primary_value = claims[0] + for index, (source_file, value) in enumerate(claims): + if index == 0 or value == primary_value: + candidate = name + owner = primary_source + else: + candidate = f"{_source_prefix(source_file, stem_counts)}_{name}" + owner = source_file + used_key = (section, candidate) + if candidate != name and used_key in reserved_names: + suffix = hashlib.sha256(source_file.encode()).hexdigest()[:10] + candidate = f"{candidate}_{suffix}" + used_key = (section, candidate) + if used_key in used and used[used_key] != value: + suffix = hashlib.sha256(source_file.encode()).hexdigest()[:10] + candidate = f"{candidate}_{suffix}" + used_key = (section, candidate) + if used_key in used and used[used_key] != value: + raise SourceContractError( + f"component namespace collision: {section}/{candidate}" + ) + used[used_key] = value + claim_key = (source_file, section, name) + assignments[claim_key] = candidate + owners[claim_key] = owner + return assignments, owners + + +def _resolve_ref_source(current_source: str, ref_path: str, source_root: Path) -> str: + if not ref_path: + return current_source + decoded = unquote(ref_path) + target = (source_root / current_source).parent.joinpath(decoded).resolve() + try: + return target.relative_to(source_root.resolve()).as_posix() + except ValueError as exc: + raise SourceContractError( + f"local $ref escapes source root in {current_source}: {ref_path}" + ) from exc + + +def _rewrite_refs( + value: Any, + *, + current_source: str, + source_root: Path, + assignments: Mapping[tuple[str, str, str], str], +) -> Any: + if isinstance(value, list): + return [ + _rewrite_refs( + item, + current_source=current_source, + source_root=source_root, + assignments=assignments, + ) + for item in value + ] + if not isinstance(value, dict): + return value + + result: dict[str, Any] = {} + for key, child in value.items(): + if key == "$ref" and isinstance(child, str): + parsed = urlsplit(child) + if parsed.scheme or parsed.netloc: + raise SourceContractError( + f"external $ref cannot be bundled in {current_source}: {child}" + ) + fragment = unquote(parsed.fragment) + parts = fragment.split("/") + if len(parts) >= 4 and parts[:2] == ["", "components"]: + section, name = parts[2], parts[3] + target_source = _resolve_ref_source( + current_source, parsed.path, source_root + ) + bundled_name = assignments.get((target_source, section, name)) + if bundled_name is None: + raise SourceContractError( + f"unresolved component $ref in {current_source}: {child}" + ) + suffix = "/".join(parts[4:]) + rewritten = f"#/components/{section}/{bundled_name}" + result[key] = f"{rewritten}/{suffix}" if suffix else rewritten + elif parsed.path: + raise SourceContractError( + f"cross-file $ref must target a component in {current_source}: {child}" + ) + else: + result[key] = child + else: + result[key] = _rewrite_refs( + child, + current_source=current_source, + source_root=source_root, + assignments=assignments, + ) + return result + + +def _merge_unique_mapping( + target: dict[str, Any], incoming: Mapping[str, Any], *, label: str, source_file: str +) -> None: + for key, value in incoming.items(): + if key in target and target[key] != value: + raise SourceContractError( + f"conflicting {label} {key!r} while merging {source_file}" + ) + target[key] = value + + +def _merge_paths( + target: dict[str, Any], incoming: Mapping[str, Any], *, source_file: str +) -> None: + for api_path, path_item in incoming.items(): + if api_path not in target: + target[api_path] = path_item + continue + current = target[api_path] + if not isinstance(current, dict) or not isinstance(path_item, dict): + raise SourceContractError( + f"conflicting path {api_path!r} while merging {source_file}" + ) + _merge_unique_mapping( + current, + path_item, + label=f"path item {api_path}", + source_file=source_file, + ) + + +def build_bundle(source_root: Path) -> tuple[dict[str, Any], dict[str, Any]]: + """Return a deterministic complete bundle and a collision/parity report.""" + + source_root = Path(source_root).resolve() + validate_source_tree(source_root) + inventory = build_source_inventory(source_root) + assert_no_method_path_conflicts(inventory) + + documents = { + source_file: strip_operation_code_samples(document) + for source_file, document in iter_source_documents(source_root) + } + if not documents: + raise SourceContractError("canonical source tree contains no documents") + + assignments, component_owners = _assign_components(documents) + first_source = next(iter(documents)) + first = documents[first_source] + bundle = { + key: copy.deepcopy(value) + for key, value in first.items() + if key not in {"tags", "paths", "components", "webhooks"} + } + bundle.update({"tags": [], "paths": {}, "components": {}, "webhooks": {}}) + for section in COMPONENT_SECTIONS: + bundle["components"][section] = {} + + collision_report: list[dict[str, str]] = [] + for source_file, document in documents.items(): + tags = document.get("tags", []) + if not isinstance(tags, list): + raise SourceContractError(f"tags must be an array: {source_file}") + for tag in tags: + if tag not in bundle["tags"]: + bundle["tags"].append(copy.deepcopy(tag)) + + paths = _rewrite_refs( + document.get("paths", {}), + current_source=source_file, + source_root=source_root, + assignments=assignments, + ) + webhooks = _rewrite_refs( + document.get("webhooks", {}), + current_source=source_file, + source_root=source_root, + assignments=assignments, + ) + if not isinstance(paths, dict) or not isinstance(webhooks, dict): + raise SourceContractError(f"paths/webhooks must be objects: {source_file}") + _merge_paths(bundle["paths"], paths, source_file=source_file) + _merge_unique_mapping( + bundle["webhooks"], webhooks, label="webhook", source_file=source_file + ) + + components = document.get("components", {}) + for section in COMPONENT_SECTIONS: + entries = components.get(section, {}) + for original_name, value in entries.items(): + claim_key = (source_file, section, original_name) + bundled_name = assignments[claim_key] + rewritten_value = _rewrite_refs( + value, + current_source=component_owners[claim_key], + source_root=source_root, + assignments=assignments, + ) + _merge_unique_mapping( + bundle["components"][section], + {bundled_name: rewritten_value}, + label=f"component {section}", + source_file=source_file, + ) + if bundled_name != original_name: + collision_report.append( + { + "source_file": source_file, + "section": section, + "original_name": original_name, + "bundled_name": bundled_name, + } + ) + + bundle["tags"] = sorted(bundle["tags"], key=lambda tag: tag.get("name", "")) + bundle["paths"] = dict(sorted(bundle["paths"].items())) + for path, path_item in bundle["paths"].items(): + if isinstance(path_item, dict): + bundle["paths"][path] = dict(sorted(path_item.items())) + bundle["webhooks"] = dict(sorted(bundle["webhooks"].items())) + bundle["components"] = { + section: dict(sorted(entries.items())) + for section, entries in sorted(bundle["components"].items()) + } + + operation_count = sum( + 1 + for identity in inventory.operations + if not identity.path.startswith("webhook:") + ) + webhook_count = len(inventory.operations) - operation_count + report = { + "source_file_count": len(documents), + "operation_count": operation_count, + "webhook_operation_count": webhook_count, + "component_collision_count": len(collision_report), + "component_collisions": sorted( + collision_report, + key=lambda item: ( + item["section"], + item["original_name"], + item["source_file"], + ), + ), + "sdk_bundle_sha256": hashlib.sha256( + canonical_semantic_bytes(bundle) + ).hexdigest(), + } + return bundle, report + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source_root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + + bundle, report = build_bundle(args.source_root) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(bundle, indent=2, ensure_ascii=True) + "\n", encoding="utf-8" + ) + args.report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/openapi-source/source_contract.py b/.github/scripts/openapi-source/source_contract.py new file mode 100644 index 0000000..0d6b102 --- /dev/null +++ b/.github/scripts/openapi-source/source_contract.py @@ -0,0 +1,429 @@ +"""Deterministic contracts for canonical OpenAPI source trees.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +from collections import defaultdict +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlsplit + +import yaml + +HTTP_METHODS = frozenset( + {"get", "put", "post", "delete", "options", "head", "patch", "trace"} +) + + +class SourceContractError(ValueError): + """Canonical source violates a safety or identity invariant.""" + + +class _UniqueKeyLoader(yaml.SafeLoader): + """Safe YAML loader that refuses silent mapping-key overwrites.""" + + +def _construct_unique_mapping( + loader: _UniqueKeyLoader, node: yaml.nodes.MappingNode, deep: bool = False +) -> dict[Any, Any]: + loader.flatten_mapping(node) + mapping: dict[Any, Any] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as exc: + raise SourceContractError(f"unhashable mapping key: {key!r}") from exc + if duplicate: + raise SourceContractError(f"duplicate mapping key: {key!r}") + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + _construct_unique_mapping, +) + +# OpenAPI uses JSON's scalar model even when serialized as YAML. PyYAML's YAML +# 1.1 resolver otherwise turns values such as ``yes`` and dates into Python-only +# types and response-code keys into integers. Restrict implicit resolution to +# JSON-compatible scalar syntax. +for first_character, resolvers in tuple( + _UniqueKeyLoader.yaml_implicit_resolvers.items() +): + _UniqueKeyLoader.yaml_implicit_resolvers[first_character] = [ + (tag, expression) + for tag, expression in resolvers + if tag + not in { + "tag:yaml.org,2002:bool", + "tag:yaml.org,2002:float", + "tag:yaml.org,2002:int", + "tag:yaml.org,2002:timestamp", + } + ] + +_UniqueKeyLoader.add_implicit_resolver( + "tag:yaml.org,2002:bool", re.compile(r"^(?:true|false)$"), list("tf") +) +_UniqueKeyLoader.add_implicit_resolver( + "tag:yaml.org,2002:int", + re.compile(r"^-?(?:0|[1-9][0-9]*)$"), + list("-0123456789"), +) +_UniqueKeyLoader.add_implicit_resolver( + "tag:yaml.org,2002:float", + re.compile( + r"^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)$" + r"|^-?(?:0|[1-9][0-9]*)\.[0-9]+$" + ), + list("-0123456789"), +) + + +@dataclass(frozen=True, order=True) +class OperationIdentity: + source_file: str + path: str + method: str + operation_id: str | None + + +@dataclass(frozen=True) +class SourceInventory: + operations: tuple[OperationIdentity, ...] + duplicate_operation_ids: dict[str, tuple[OperationIdentity, ...]] + method_path_conflicts: dict[tuple[str, str], tuple[OperationIdentity, ...]] + + +def _walk_refs(value: Any) -> Iterator[str]: + if isinstance(value, dict): + for key, child in value.items(): + if key == "$ref" and isinstance(child, str): + yield child + yield from _walk_refs(child) + elif isinstance(value, list): + for child in value: + yield from _walk_refs(child) + + +def strip_operation_code_samples(document: Mapping[str, Any]) -> dict[str, Any]: + """Return a deep copy without operation-level ``x-codeSamples`` fields.""" + + normalized = copy.deepcopy(document) + paths = normalized.get("paths") + if not isinstance(paths, dict): + return normalized + + for path_item in paths.values(): + if not isinstance(path_item, dict): + continue + for method, operation in path_item.items(): + if method.lower() in HTTP_METHODS and isinstance(operation, dict): + operation.pop("x-codeSamples", None) + + return normalized + + +def canonical_semantic_bytes(document: Mapping[str, Any]) -> bytes: + """Serialize parsed OpenAPI deterministically for semantic checksums.""" + + return json.dumps( + document, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _source_files(root: Path) -> Iterator[Path]: + for path in sorted( + root.rglob("*"), key=lambda item: item.relative_to(root).as_posix() + ): + if path.is_symlink(): + raise SourceContractError( + f"symbolic link is not allowed in source tree: {path}" + ) + if path.is_file() and path.suffix.lower() in {".json", ".yaml", ".yml"}: + yield path + + +def _update_framed(digest: Any, value: bytes) -> None: + digest.update(len(value).to_bytes(8, byteorder="big")) + digest.update(value) + + +def raw_tree_sha256(root: Path) -> str: + """Hash supported source paths and their exact bytes deterministically.""" + + root = Path(root).resolve() + digest = hashlib.sha256() + for path in _source_files(root): + relative = path.relative_to(root).as_posix().encode("utf-8") + _update_framed(digest, relative) + _update_framed(digest, path.read_bytes()) + return digest.hexdigest() + + +def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + mapping: dict[str, Any] = {} + for key, value in pairs: + if key in mapping: + raise SourceContractError(f"duplicate mapping key: {key!r}") + mapping[key] = value + return mapping + + +def _reject_json_constant(value: str) -> None: + raise SourceContractError(f"invalid JSON constant: {value}") + + +def _load_document(path: Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") + try: + if path.suffix.lower() == ".json": + document = json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_json_constant, + ) + else: + document = yaml.load(text, Loader=_UniqueKeyLoader) + except SourceContractError as exc: + raise SourceContractError(f"{exc} in {path}") from exc + except (json.JSONDecodeError, yaml.YAMLError) as exc: + raise SourceContractError(f"invalid YAML/JSON in {path}: {exc}") from exc + if not isinstance(document, dict): + raise SourceContractError(f"OpenAPI source must be an object: {path}") + return _stringify_mapping_keys(document, path) + + +def _stringify_mapping_keys(value: Any, path: Path) -> Any: + """Normalize YAML mapping keys to JSON object keys without hiding collisions.""" + + if isinstance(value, list): + return [_stringify_mapping_keys(item, path) for item in value] + if not isinstance(value, dict): + return value + + normalized: dict[str, Any] = {} + for key, child in value.items(): + if not isinstance(key, (str, int, float, bool)) and key is not None: + raise SourceContractError( + f"mapping key is not JSON-compatible in {path}: {key!r}" + ) + normalized_key = str(key).lower() if isinstance(key, bool) else str(key) + if normalized_key in normalized: + raise SourceContractError( + f"duplicate mapping key after JSON normalization in {path}: " + f"{normalized_key!r}" + ) + normalized[normalized_key] = _stringify_mapping_keys(child, path) + return normalized + + +def load_document(path: Path) -> dict[str, Any]: + """Safely parse one JSON or YAML OpenAPI document.""" + + return _load_document(Path(path)) + + +def source_files(root: Path) -> tuple[Path, ...]: + """Return supported source files in stable source-relative order.""" + + root = Path(root).resolve() + return tuple(_source_files(root)) + + +def iter_source_documents(root: Path) -> Iterator[tuple[str, dict[str, Any]]]: + """Yield source-relative paths and safely parsed OpenAPI documents.""" + + root = Path(root).resolve() + for path in _source_files(root): + yield path.relative_to(root).as_posix(), _load_document(path) + + +def build_source_inventory(root: Path) -> SourceInventory: + """Inventory stable source-file/path/method operation identities.""" + + root = Path(root).resolve() + operations: list[OperationIdentity] = [] + by_operation_id: dict[str, list[OperationIdentity]] = defaultdict(list) + by_method_path: dict[tuple[str, str], list[OperationIdentity]] = defaultdict(list) + + for source_path in _source_files(root): + document = _load_document(source_path) + paths = document.get("paths", {}) + if not isinstance(paths, dict): + raise SourceContractError(f"paths must be an object: {source_path}") + for api_path, path_item in paths.items(): + if not isinstance(api_path, str) or not isinstance(path_item, dict): + raise SourceContractError( + f"invalid path item in {source_path}: {api_path!r}" + ) + for method, operation in path_item.items(): + normalized_method = str(method).lower() + if normalized_method not in HTTP_METHODS: + continue + if not isinstance(operation, dict): + raise SourceContractError( + f"operation must be an object in {source_path}: " + f"{normalized_method.upper()} {api_path}" + ) + operation_id_value = operation.get("operationId") + operation_id = ( + operation_id_value + if isinstance(operation_id_value, str) and operation_id_value + else None + ) + identity = OperationIdentity( + source_file=source_path.relative_to(root).as_posix(), + path=api_path, + method=normalized_method, + operation_id=operation_id, + ) + operations.append(identity) + by_method_path[(api_path, normalized_method)].append(identity) + if operation_id is not None: + by_operation_id[operation_id].append(identity) + + webhooks = document.get("webhooks", {}) + if not isinstance(webhooks, dict): + raise SourceContractError(f"webhooks must be an object: {source_path}") + for webhook_name, path_item in webhooks.items(): + if not isinstance(webhook_name, str) or not isinstance(path_item, dict): + raise SourceContractError( + f"invalid webhook item in {source_path}: {webhook_name!r}" + ) + for method, operation in path_item.items(): + normalized_method = str(method).lower() + if normalized_method not in HTTP_METHODS: + continue + if not isinstance(operation, dict): + raise SourceContractError( + f"webhook operation must be an object in {source_path}: " + f"{normalized_method.upper()} {webhook_name}" + ) + operation_id_value = operation.get("operationId") + operation_id = ( + operation_id_value + if isinstance(operation_id_value, str) and operation_id_value + else None + ) + identity = OperationIdentity( + source_file=source_path.relative_to(root).as_posix(), + path=f"webhook:{webhook_name}", + method=normalized_method, + operation_id=operation_id, + ) + operations.append(identity) + if operation_id is not None: + by_operation_id[operation_id].append(identity) + + sorted_operations = tuple(sorted(operations)) + duplicate_operation_ids = { + operation_id: tuple(sorted(matches)) + for operation_id, matches in sorted(by_operation_id.items()) + if len(matches) > 1 + } + method_path_conflicts = { + identity: tuple(sorted(matches)) + for identity, matches in sorted(by_method_path.items()) + if len({match.source_file for match in matches}) > 1 + } + return SourceInventory( + operations=sorted_operations, + duplicate_operation_ids=duplicate_operation_ids, + method_path_conflicts=method_path_conflicts, + ) + + +def assert_no_method_path_conflicts(inventory: SourceInventory) -> None: + """Fail when multiple canonical source files claim the same HTTP operation.""" + + if not inventory.method_path_conflicts: + return + details = [] + for (api_path, method), matches in inventory.method_path_conflicts.items(): + sources = ", ".join(match.source_file for match in matches) + details.append(f"{method.upper()} {api_path} ({sources})") + raise SourceContractError("cross-file method/path conflicts: " + "; ".join(details)) + + +def validate_source_tree(root: Path) -> None: + """Reject local references that can escape the canonical source root.""" + + root = Path(root).resolve() + for source_path in _source_files(root): + document = _load_document(source_path) + for ref in _walk_refs(document): + parsed = urlsplit(ref) + if parsed.scheme or parsed.netloc: + raise SourceContractError( + f"external $ref escapes immutable source epoch in {source_path}: {ref}" + ) + if not parsed.path: + continue + decoded_path = unquote(parsed.path) + if "\\" in decoded_path: + raise SourceContractError( + f"local $ref uses unsafe path separators in {source_path}: {ref}" + ) + candidate = (source_path.parent / decoded_path).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise SourceContractError( + f"local $ref escapes source root in {source_path}: {ref}" + ) from exc + if not candidate.is_file(): + raise SourceContractError( + f"local $ref target does not exist in {source_path}: {ref}" + ) + + +def semantic_tree_sha256(root: Path, *, strip_samples: bool) -> str: + """Hash parsed source semantics while retaining each relative source path.""" + + root = Path(root).resolve() + digest = hashlib.sha256() + for path in _source_files(root): + document = _load_document(path) + if strip_samples: + document = strip_operation_code_samples(document) + _update_framed(digest, path.relative_to(root).as_posix().encode("utf-8")) + _update_framed(digest, canonical_semantic_bytes(document)) + return digest.hexdigest() + + +def normalized_semantic_sha256(document: Mapping[str, Any]) -> str: + """Hash OpenAPI semantics after removing CI-owned code samples.""" + + normalized = strip_operation_code_samples(document) + return hashlib.sha256(canonical_semantic_bytes(normalized)).hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + checksum = subparsers.add_parser("checksum") + checksum.add_argument("root", type=Path) + checksum.add_argument("--strip-samples", action="store_true") + args = parser.parse_args() + if args.command == "checksum": + print(semantic_tree_sha256(args.root, strip_samples=args.strip_samples)) + return 0 + parser.error(f"unsupported command: {args.command}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/openapi-source/test_build_public_mirror.py b/.github/scripts/openapi-source/test_build_public_mirror.py new file mode 100644 index 0000000..67a6c4d --- /dev/null +++ b/.github/scripts/openapi-source/test_build_public_mirror.py @@ -0,0 +1,29 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from build_public_mirror import strip_hidden_operations + + +class PublicMirrorTest(unittest.TestCase): + def test_hidden_operations_are_removed_without_dropping_visible_siblings(self): + source = { + "paths": { + "/mixed": { + "get": {"operationId": "visible"}, + "post": {"operationId": "hidden", "x-hidden": True}, + }, + "/hidden": {"delete": {"operationId": "hiddenOnly", "x-hidden": True}}, + } + } + public, removed = strip_hidden_operations(source) + self.assertEqual(removed, 2) + self.assertIn("get", public["paths"]["/mixed"]) + self.assertNotIn("post", public["paths"]["/mixed"]) + self.assertNotIn("/hidden", public["paths"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/openapi-source/test_bundle_canonical_source.py b/.github/scripts/openapi-source/test_bundle_canonical_source.py new file mode 100644 index 0000000..618cd35 --- /dev/null +++ b/.github/scripts/openapi-source/test_bundle_canonical_source.py @@ -0,0 +1,155 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPTS)) + +from bundle_canonical_source import build_bundle +from source_contract import SourceContractError + + +def spec(title, path, operation_id, schema): + return { + "openapi": "3.1.0", + "info": {"title": title, "version": "1.0.0"}, + "paths": { + path: { + "get": { + "operationId": operation_id, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/Resource"} + } + }, + } + }, + "x-codeSamples": [{"lang": "Python", "source": "ignored"}], + } + } + }, + "components": {"schemas": {"Resource": schema}}, + } + + +class CanonicalBundleTest(unittest.TestCase): + def test_rewrites_conflicting_component_refs_without_overwrite(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "a.json").write_text( + json.dumps(spec("A", "/a", "getA", {"type": "string"})) + ) + (root / "b.json").write_text( + json.dumps(spec("B", "/b", "getB", {"type": "integer"})) + ) + + bundle, report = build_bundle(root) + + self.assertEqual(bundle["components"]["schemas"]["Resource"]["type"], "string") + self.assertEqual( + bundle["components"]["schemas"]["b_Resource"]["type"], "integer" + ) + self.assertEqual( + bundle["paths"]["/a"]["get"]["responses"]["200"]["content"][ + "application/json" + ]["schema"]["$ref"], + "#/components/schemas/Resource", + ) + self.assertEqual( + bundle["paths"]["/b"]["get"]["responses"]["200"]["content"][ + "application/json" + ]["schema"]["$ref"], + "#/components/schemas/b_Resource", + ) + self.assertNotIn("x-codeSamples", bundle["paths"]["/a"]["get"]) + self.assertEqual(report["component_collision_count"], 1) + + def test_identical_components_share_the_unqualified_name(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + schema = {"type": "string"} + (root / "a.json").write_text(json.dumps(spec("A", "/a", "getA", schema))) + (root / "b.json").write_text(json.dumps(spec("B", "/b", "getB", schema))) + + bundle, report = build_bundle(root) + + self.assertEqual(list(bundle["components"]["schemas"]), ["Resource"]) + self.assertEqual(report["component_collision_count"], 0) + + def test_duplicate_file_stems_receive_stable_path_prefixes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "first").mkdir() + (root / "second").mkdir() + (root / "first" / "api.json").write_text( + json.dumps(spec("A", "/a", "getA", {"type": "string"})) + ) + (root / "second" / "api.json").write_text( + json.dumps(spec("B", "/b", "getB", {"type": "integer"})) + ) + + bundle, _ = build_bundle(root) + + self.assertIn("second_api_Resource", bundle["components"]["schemas"]) + + def test_cross_file_component_ref_resolves_to_target_assignment(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "models").mkdir() + target = spec("Target", "/target", "getTarget", {"type": "string"}) + source = spec("Source", "/source", "getSource", {"type": "integer"}) + source["paths"]["/source"]["get"]["responses"]["200"]["content"][ + "application/json" + ]["schema"]["$ref"] = "models/target.json#/components/schemas/Resource" + (root / "models" / "target.json").write_text(json.dumps(target)) + (root / "source.json").write_text(json.dumps(source)) + + bundle, _ = build_bundle(root) + + ref = bundle["paths"]["/source"]["get"]["responses"]["200"]["content"][ + "application/json" + ]["schema"]["$ref"] + self.assertEqual(ref, "#/components/schemas/Resource") + + def test_method_path_conflict_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "a.json").write_text( + json.dumps(spec("A", "/same", "getA", {"type": "string"})) + ) + (root / "b.json").write_text( + json.dumps(spec("B", "/same", "getB", {"type": "integer"})) + ) + + with self.assertRaisesRegex(SourceContractError, "method/path conflicts"): + build_bundle(root) + + def test_output_is_independent_of_temporary_root(self): + documents = [ + ("z.json", spec("Z", "/z", "getZ", {"type": "integer"})), + ("a.json", spec("A", "/a", "getA", {"type": "string"})), + ] + with ( + tempfile.TemporaryDirectory() as first, + tempfile.TemporaryDirectory() as second, + ): + first_root, second_root = Path(first), Path(second) + for name, document in documents: + (first_root / name).write_text(json.dumps(document)) + for name, document in reversed(documents): + (second_root / name).write_text(json.dumps(document)) + + first_bundle, first_report = build_bundle(first_root) + second_bundle, second_report = build_bundle(second_root) + + self.assertEqual(first_bundle, second_bundle) + self.assertEqual(first_report, second_report) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/sync-canonical-openapi.yml b/.github/workflows/sync-canonical-openapi.yml new file mode 100644 index 0000000..f4a12d0 --- /dev/null +++ b/.github/workflows/sync-canonical-openapi.yml @@ -0,0 +1,147 @@ +name: Sync canonical public OpenAPI mirror + +on: + pull_request: + paths: + - '.github/workflows/sync-canonical-openapi.yml' + - '.github/scripts/openapi-source/**' + repository_dispatch: + types: [canonical-openapi-source] + workflow_dispatch: + inputs: + source_sha: + required: true + type: string + normalized_tree_sha256: + required: true + type: string + sdk_bundle_sha256: + required: true + type: string + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-canonical-public-openapi + cancel-in-progress: false + +jobs: + validate-consumer: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install pyyaml + - run: python -m unittest discover -s .github/scripts/openapi-source -p 'test_*.py' -v + + sync: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: master + fetch-depth: 0 + token: ${{ secrets.OPENAPI_DOCS_SYNC_TOKEN }} + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install pyyaml + + - name: Validate immutable dispatch + id: source + env: + GH_TOKEN: ${{ secrets.OPENAPI_DOCS_SYNC_TOKEN }} + EVENT_REPOSITORY: ${{ github.event.client_payload.source_repository }} + EVENT_SHA: ${{ github.event.client_payload.source_sha }} + EVENT_NORMALIZED: ${{ github.event.client_payload.normalized_tree_sha256 }} + EVENT_BUNDLE: ${{ github.event.client_payload.sdk_bundle_sha256 }} + MANUAL_SHA: ${{ inputs.source_sha }} + MANUAL_NORMALIZED: ${{ inputs.normalized_tree_sha256 }} + MANUAL_BUNDLE: ${{ inputs.sdk_bundle_sha256 }} + run: | + # shellcheck disable=SC2129 + set -euo pipefail + repository="${EVENT_REPOSITORY:-team-telnyx/developer-docs-mintlify}" + source_sha="${EVENT_SHA:-${MANUAL_SHA:-}}" + normalized="${EVENT_NORMALIZED:-${MANUAL_NORMALIZED:-}}" + bundle="${EVENT_BUNDLE:-${MANUAL_BUNDLE:-}}" + test "$repository" = 'team-telnyx/developer-docs-mintlify' + [[ "$source_sha" =~ ^[0-9a-f]{40}$ ]] + [[ "$normalized" =~ ^[0-9a-f]{64}$ ]] + [[ "$bundle" =~ ^[0-9a-f]{64}$ ]] + gh api "repos/$repository/commits/$source_sha" >/dev/null + current="$(jq -r '.source_sha // empty' openapi/direct-source-provenance.json 2>/dev/null || true)" + if [[ "$current" == "$source_sha" ]]; then + noop=true + elif [[ -n "$current" ]]; then + test "$(gh api "repos/$repository/compare/$current...$source_sha" --jq .status)" = 'ahead' + noop=false + else + noop=false + fi + { + echo "noop=$noop" + echo "repository=$repository" + echo "source_sha=$source_sha" + echo "normalized=$normalized" + echo "bundle=$bundle" + } >> "$GITHUB_OUTPUT" + + - name: Build and attest public mirror + if: steps.source.outputs.noop != 'true' + env: + GH_TOKEN: ${{ secrets.OPENAPI_DOCS_SYNC_TOKEN }} + SOURCE_REPOSITORY: ${{ steps.source.outputs.repository }} + SOURCE_SHA: ${{ steps.source.outputs.source_sha }} + EXPECTED_NORMALIZED: ${{ steps.source.outputs.normalized }} + EXPECTED_BUNDLE: ${{ steps.source.outputs.bundle }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/upstream" + gh api "repos/$SOURCE_REPOSITORY/tarball/$SOURCE_SHA" > "$RUNNER_TEMP/source.tar.gz" + tar -xzf "$RUNNER_TEMP/source.tar.gz" --strip-components=1 -C "$RUNNER_TEMP/upstream" + root="$RUNNER_TEMP/upstream/openapi/source/external" + test "$(python .github/scripts/openapi-source/source_contract.py checksum "$root" --strip-samples)" = "$EXPECTED_NORMALIZED" + python .github/scripts/openapi-source/bundle_canonical_source.py \ + "$root" --output "$RUNNER_TEMP/bundle.json" --report "$RUNNER_TEMP/bundle-report.json" + test "$(sha256sum "$RUNNER_TEMP/bundle.json" | cut -d' ' -f1)" = "$EXPECTED_BUNDLE" + python .github/scripts/openapi-source/build_public_mirror.py \ + "$root" \ + --json-output openapi/spec3.json \ + --yaml-output openapi/spec3.yml \ + --report "$RUNNER_TEMP/public-report.json" + jq -n \ + --arg source_repository "$SOURCE_REPOSITORY" \ + --arg source_sha "$SOURCE_SHA" \ + --arg normalized "$EXPECTED_NORMALIZED" \ + --arg bundle "$EXPECTED_BUNDLE" \ + --arg public "$(jq -r .public_semantic_sha256 "$RUNNER_TEMP/public-report.json")" \ + '{schema_version:1,source_repository:$source_repository,source_sha:$source_sha,source_root:"openapi/source/external",normalized_tree_sha256:$normalized,sdk_bundle_sha256:$bundle,public_semantic_sha256:$public}' \ + > openapi/direct-source-provenance.json + + - name: Create immutable public-mirror PR + if: steps.source.outputs.noop != 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.OPENAPI_DOCS_SYNC_TOKEN }} + base: master + branch: chore/sync-canonical-openapi + delete-branch: true + add-paths: | + openapi/spec3.json + openapi/spec3.yml + openapi/direct-source-provenance.json + commit-message: 'chore: sync canonical public OpenAPI mirror' + title: 'chore: sync canonical public OpenAPI mirror' + labels: automated + body: | + Generated public mirror from immutable + `team-telnyx/developer-docs-mintlify@${{ steps.source.outputs.source_sha }}`. + Hidden operations are excluded. Source and bundle checksums were rebuilt + with trusted default-branch consumer code; stale or divergent epochs fail closed.