[Feature] Modernization: pydantic types generated from the OpenAPI spec and GraphQL schema

.github/workflows/ci.yml

@@ -22,6 +22,8 @@ jobs:
 
     - run: make install
     - run: make lint
+    # python_anvil/types must be regenerated when schema/ changes
+    - run: make check-generated
 
   test:
     name: Test

.github/workflows/sync-api-types.yml

@@ -0,0 +1,158 @@
+name: Sync API Types
+
+# Refreshes schema/ from the live Anvil API and the published @anvilco/types,
+# regenerates python_anvil/types, and opens one PR when any of it changed.
+# Runs after a production rollout completes: Foundry's postsync step sends
+# `anvil-production-deploy` to every repo in its --webhook-repos list
+# (anvil-infra: k8s/anvil/base/foundry/postsync.yaml). That dispatch is
+# best-effort with no retry, so the weekly cron is a backstop for a dropped one.
+# anvil-js's sync-api-types.yml does the same for @anvilco/types.
+
+on:
+  repository_dispatch:
+    types: [anvil-production-deploy]
+  schedule:
+    - cron: '0 10 * * 1' # Every Monday at 10am UTC, after anvil-js's 9am sync
+  workflow_dispatch:
+
+# BRANCH is force-pushed, so never let two runs race on it.
+concurrency: sync-api-types
+
+env:
+  BRANCH: chore/sync-api-types
+  SCHEMA_DIR: schema
+  TYPES_DIR: python_anvil/types
+
+jobs:
+  sync-api-types:
+    name: Sync API Types
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      # Always master, so a manual run from another branch can't put that
+      # branch's commits in the PR. No persisted credentials: CI_GITHUB_TOKEN
+      # is only handed to git and gh in the steps that push, not to the
+      # install and the generators.
+      - uses: actions/checkout@v7
+        with:
+          ref: master
+          persist-credentials: false
+
+      - run: pipx install poetry
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version-file: '.python-version'
+          cache: 'poetry'
+
+      - run: make install
+
+      # One step, so the types are never regenerated from a spec that failed
+      # to fetch, and a failed generation is discarded along with the specs.
+      - name: Fetch the specs and generate the types
+        id: generate
+        continue-on-error: true
+        run: |
+          make update-schema
+          make generate
+
+      - name: Discard the output if that failed
+        if: steps.generate.outcome != 'success'
+        run: git checkout -- "$SCHEMA_DIR" "$TYPES_DIR"
+
+      - name: Check for changes
+        id: changes
+        run: |
+          echo "changed=$(git diff --quiet -- "$SCHEMA_DIR" "$TYPES_DIR" && echo false || echo true)" >> "$GITHUB_OUTPUT"
+          git diff --stat
+
+      - name: Open or update PR
+        if: steps.changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }}
+          EVENT_NAME: ${{ github.event_name }}
+          DEPLOY_VERSION: ${{ github.event.client_payload.version }}
+          DEPLOY_PR_URL: ${{ github.event.client_payload.pullRequest.url }}
+          DEPLOYED_AT: ${{ github.event.client_payload.deployedAt }}
+        run: |
+          # Not GITHUB_TOKEN: pushes and PRs made with it don't trigger
+          # workflows, so the PR would never get its CI checks.
+          git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
+
+          # A deploy that didn't move the API again would otherwise force-push
+          # an identical commit, re-running CI and discarding any edits made
+          # on the branch.
+          PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty')
+          if [ -n "$PR" ] && git fetch -q --depth=1 origin "$BRANCH" && git diff --quiet FETCH_HEAD -- "$SCHEMA_DIR" "$TYPES_DIR"; then
+            echo "PR #$PR already has this output"
+            exit 0
+          fi
+
+          # Note the sync under Unreleased in the changelog, once per release
+          python3 - <<'PY'
+          import re
+          from pathlib import Path
+
+          path = Path("CHANGELOG.md")
+          text = path.read_text(encoding="utf-8")
+          note = "- Sync the generated API types in `python_anvil.types` with the production API."
+          match = re.search(r"^## \[Unreleased\]\n(?P<body>.*?)(?=^## \[|\Z)", text, re.M | re.S)
+          if match and note not in match["body"]:
+              body = match["body"].rstrip("\n")
+              section = f"{body}\n{note}\n\n" if body else f"\n{note}\n\n"
+              text = text[: match.start("body")] + section + text[match.end("body") :]
+              path.write_text(text, encoding="utf-8")
+          PY
+
+          if [ "$EVENT_NAME" = repository_dispatch ]; then
+            TRIGGER="Anvil production deploy \`$DEPLOY_VERSION\` ($DEPLOY_PR_URL), deployed $DEPLOYED_AT."
+          else
+            TRIGGER="\`$EVENT_NAME\` run."
+          fi
+          {
+            echo "Automated sync of the generated types from the production Anvil API. Triggered by $TRIGGER"
+            echo
+            echo "- \`$SCHEMA_DIR/\`: the GraphQL schema (from the published \`@anvilco/types\`) and the OpenAPI spec"
+            echo "- \`$TYPES_DIR/\`: regenerated from them with \`make generate\`"
+            echo
+            echo "If a type or field was removed or renamed, note it under Unreleased in CHANGELOG.md as a breaking change before merging."
+            echo
+            echo "A later deploy that changes these files again rebuilds this branch from \`master\`, replacing any edits made here."
+          } > "$RUNNER_TEMP/pr-body.md"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A -- "$SCHEMA_DIR" "$TYPES_DIR" CHANGELOG.md
+          git commit -m "Sync the generated API types from production"
+          git push --force origin "$BRANCH"
+
+          if [ -n "$PR" ]; then
+            gh pr edit "$PR" --body-file "$RUNNER_TEMP/pr-body.md"
+          else
+            gh pr create \
+              --title "Sync the generated API types from production" \
+              --body-file "$RUNNER_TEMP/pr-body.md" \
+              --base master \
+              --head "$BRANCH"
+          fi
+
+      # Production moved back to match master (e.g. a rollback before the PR
+      # merged), so an open sync PR is now wrong.
+      - name: Close stale PR
+        if: steps.changes.outputs.changed == 'false' && steps.generate.outcome == 'success'
+        env:
+          GH_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }}
+        run: |
+          PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty')
+          if [ -n "$PR" ]; then
+            gh pr close "$PR" --delete-branch --comment "Production now matches master, so this sync is no longer needed."
+          fi
+
+      - name: Fail if the generator failed
+        if: steps.generate.outcome != 'success'
+        run: |
+          echo "::error::Fetching the specs or generating the types failed. See the step above."
+          exit 1

CHANGELOG.md

@@ -30,8 +30,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The `createEtchPacket` mutation gains `organizationEid` and `allowUpdates`, and the mutations and default response fields now match node-anvil's exactly. `forgeSubmit`'s default fields drop `resolvedPayload` and `weldData.status`, and gain `continueURL` and `payloadValue`; pass `response_query` to select others.
 - `forge_submit` with a `webhook_url` works. `ForgeSubmitPayload.webhook_url` was a pydantic `HttpUrl`, which the JSON encoder rejected, so it failed before sending; it is a `str` now.
 - `fill_pdf` sends `version_number=0` (it used to drop it), and `generate_etch_signing_url` is also available as `generate_etch_sign_url`.
+- `Anvil()` with no key reads `ANVIL_API_KEY` from the environment, the way the examples and the CLI do. The client keeps `api_key` and `access_token` as attributes.
+- Every method takes a `timeout` keyword argument, overriding the client's timeout for that request.
+- `create_etch_packet` and `forge_submit` take the payload as a JSON string too, like `fill_pdf` and `generate_pdf`, and their dicts may hold any kind of file (a template reference, Markdown, HTML or an upload) or model instances, with snake_case or camelCase keys. `query` and `mutate` take `variables` as a JSON string. A `response_query` may be given with or without braces. An `in-person` signer is accepted. An uploaded open file is sent under its own filename rather than `document.pdf`.
+- `GeneratePDFPayload` extends the generated request type like the other payload models, so `page`, `logo`, `include_timestamp` and `output_format` are typed. `utils.camel_to_snake` keeps acronyms whole now (`webhookURL` -> `webhook_url`; it gave `webhook_u_r_l`). The CLI's `cast` and `weld` without an eid exit with a usage error (code 2) instead of a traceback. `CREATE_ETCH_PACKET` and `FORGE_SUBMIT` are still exported from the builder modules; a `BaseQuery` subclass with its own `mutation` template still works.
 - The client's queries and mutations are checked against the API schema in CI, and `make update-schema` refreshes the schema from the published `@anvilco/types` package.
 - Add `test-client.py`, which runs every method against a real environment: `ANVIL_API_KEY=... make test-client`.
+- Add `python_anvil.types`: pydantic models generated from the API's OpenAPI spec and GraphQL schema, the Python counterpart of `@anvilco/types`. `types.rest` has the REST request bodies, `types.graphql` every type in the schema, and `types.operations` the arguments of every query and mutation, like `CreateEtchPacketArgs`. The client's payload models extend them, so `FillPDFPayload`, `EtchSigner`, `CreateEtchPacketPayload` and `ForgeSubmitPayload` now accept every option the API has. BREAKING: they also validate those options the way the API documents them: an enum value the API doesn't accept (like `signature_mode="wave"`), or a number out of its documented range (like `token_valid_for_minutes` above a day), is a `ValidationError` before anything is sent, where 5.x passed it through for the API to reject. `font_size` and `routing_order` keep accepting any value, as they did. `ForgeSubmitPayload.webhook_url` is a plain `str` now, not an `HttpUrl`.
+- A Sync API Types workflow refreshes `schema/` and regenerates `python_anvil/types` after each production deploy, opening a PR when anything changed. `make update-schema generate` does the same by hand.
 
 ## [v5.0.3] - 2025-02-24
 

Makefile

@@ -30,9 +30,17 @@ test-client: install ## Run every method against a real environment (staging by
 	poetry run python test-client.py
 
 .PHONY: update-schema
-update-schema: install ## Refresh schema/anvil_schema.graphql from the published @anvilco/types
+update-schema: install ## Refresh schema/ from the published @anvilco/types and the API's OpenAPI spec
 	poetry run python dev/fetch_schema.py
 
+.PHONY: generate
+generate: install ## Regenerate python_anvil/types from schema/
+	poetry run python dev/generate_types.py
+
+.PHONY: check-generated
+check-generated: generate ## Fail if python_anvil/types is out of date with schema/
+	git diff --exit-code -- python_anvil/types
+
 .PHONY: build
 build: install ## Build the sdist and wheel into dist/
 	rm -rf dist

dev/fetch_schema.py

@@ -1,15 +1,20 @@
 #!/usr/bin/env python3
-"""Refreshes schema/anvil_schema.graphql from the published @anvilco/types package.
+"""Refreshes the API specs the generated types come from.
 
-@anvilco/types (https://www.npmjs.com/package/@anvilco/types) ships the Anvil
-GraphQL schema, regenerated from production after every deploy. node-anvil
-validates its operations against that copy, and so does this client: using
-the same source keeps the two clients in step, and needs no API key.
+- schema/anvil_schema.graphql, from the published @anvilco/types npm package
+  (https://www.npmjs.com/package/@anvilco/types). It ships the Anvil GraphQL
+  schema, regenerated from production after every deploy. node-anvil
+  validates its operations against that copy, and so does this client:
+  using the same source keeps the two clients in step.
+- schema/openapi.json, the API's own OpenAPI spec, from production.
+
+Neither needs an API key. Then run `dev/generate_types.py` (`make generate`)
+to regenerate python_anvil/types from them.
 
 Usage:
-    poetry run python dev/fetch_schema.py [version]
+    poetry run python dev/fetch_schema.py [@anvilco/types version]
 
-Defaults to the latest version. On any failure the existing file is untouched.
+Defaults to the latest version. On any failure the existing files are untouched.
 """
 
 from __future__ import annotations
@@ -25,38 +30,66 @@
 PACKAGE = "@anvilco/types"
 REGISTRY = "https://registry.npmjs.org"
 SCHEMA_IN_PACKAGE = "package/schema/anvil-schema.graphql"
-OUTPUT_PATH = Path(__file__).resolve().parent.parent / "schema" / "anvil_schema.graphql"
+OPENAPI_URL = "https://app.useanvil.com/api/v1/openapi.json"
+
+SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schema"
+SCHEMA_PATH = SCHEMA_DIR / "anvil_schema.graphql"
+OPENAPI_PATH = SCHEMA_DIR / "openapi.json"
 
 
-def fetch_schema(version: str = "latest") -> tuple[str, str]:
+def fetch_graphql_schema(version: str = "latest") -> tuple[str, str]:
     """The schema SDL in the given version of @anvilco/types, and that version."""
     metadata_url = f"{REGISTRY}/{PACKAGE.replace('/', '%2F')}"
-    with urllib.request.urlopen(metadata_url) as response:
+    with urllib.request.urlopen(metadata_url, timeout=60) as response:
         metadata = json.load(response)
 
     if version == "latest":
         version = metadata["dist-tags"]["latest"]
     tarball_url = metadata["versions"][version]["dist"]["tarball"]
 
-    with urllib.request.urlopen(tarball_url) as response:
+    with urllib.request.urlopen(tarball_url, timeout=60) as response:
         tarball = response.read()
 
     with tarfile.open(fileobj=io.BytesIO(tarball), mode="r:gz") as archive:
-        member = archive.extractfile(SCHEMA_IN_PACKAGE)
+        try:
+            member = archive.extractfile(SCHEMA_IN_PACKAGE)
+        except KeyError:
+            member = None
         if member is None:
             raise FileNotFoundError(
                 f"{SCHEMA_IN_PACKAGE} is not in {PACKAGE}@{version}"
             )
         return member.read().decode("utf-8"), version
 
 
+def fetch_openapi_spec() -> str:
+    """The OpenAPI spec, pretty-printed with sorted keys so the diff is only real changes."""
+    with urllib.request.urlopen(OPENAPI_URL, timeout=60) as response:
+        spec = json.load(response)
+    if "openapi" not in spec or "paths" not in spec:
+        raise ValueError(f"{OPENAPI_URL} did not return an OpenAPI spec")
+    return json.dumps(spec, indent=2, sort_keys=True) + "\n"
+
+
 def main() -> None:
     version = sys.argv[1] if len(sys.argv) > 1 else "latest"
-    schema, version = fetch_schema(version)
-    OUTPUT_PATH.write_text(schema)
+
+    schema, version = fetch_graphql_schema(version)
+    openapi = fetch_openapi_spec()
+
+    SCHEMA_PATH.write_text(schema, encoding="utf-8")
     print(
-        f"Wrote the schema from {PACKAGE}@{version} to {OUTPUT_PATH.relative_to(Path.cwd())}"
+        f"Wrote the GraphQL schema from {PACKAGE}@{version} to {_relative(SCHEMA_PATH)}"
     )
+    OPENAPI_PATH.write_text(openapi, encoding="utf-8")
+    print(f"Wrote the OpenAPI spec from {OPENAPI_URL} to {_relative(OPENAPI_PATH)}")
+
+
+def _relative(path: Path) -> str:
+    try:
+        return str(path.relative_to(Path.cwd()))
+    except ValueError:
+        return str(path)
 
 
 if __name__ == "__main__":