diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml new file mode 100644 index 0000000..f94b3f8 --- /dev/null +++ b/.github/workflows/python-release.yml @@ -0,0 +1,130 @@ +name: python-release + +on: + workflow_dispatch: + inputs: + sdk_tag: + description: Existing immutable SDK tag (python/vX.Y.Z) + required: true + type: string + suite_tag: + description: Existing frozen Suite tag (v0.8.P), catalogued before dispatch + required: true + type: string + publish: + description: Upload verified artifacts (requires owner-configured pypi approval) + type: boolean + default: false + +permissions: {} + +concurrency: + group: python-release-${{ inputs.sdk_tag }} + cancel-in-progress: false + +jobs: + qualify: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + env: + SDK_TAG: ${{ inputs.sdk_tag }} + SUITE_TAG: ${{ inputs.suite_tag }} + GOWORK: 'off' + AGENTSUITE_GO_ORACLE_ROOT: ${{ github.workspace }}/contract-source + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.14' + - name: Validate tag namespaces before executing release source + run: | + python - <<'PY' + import os, re + assert re.fullmatch(r'python/v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)', os.environ['SDK_TAG']), 'exact SDK tag required' + assert re.fullmatch(r'v0\.8\.(0|[1-9]\d*)', os.environ['SUITE_TAG']), 'frozen v0.8 tag required' + PY + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/tags/${{ inputs.sdk_tag }} + path: sdk-source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/tags/${{ inputs.suite_tag }} + path: contract-source + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: contract-source/go.mod + - name: Verify frozen source against the pinned trusted catalog + run: | + SUITE_SHA=$(git -C contract-source rev-parse HEAD) + node scripts/schema-release.mjs check --release "$SUITE_TAG" --ref "$SUITE_SHA" --catalog-ref "$GITHUB_SHA" + - name: Qualify SDK and frozen Go output parity + run: | + python -m pip install --only-binary=google-re2 -r sdk-source/python/requirements-dev.txt ./sdk-source/python + PYTHONPATH="$GITHUB_WORKSPACE/sdk-source/python/src" python -m pytest sdk-source/python/tests -q + PYTHONPATH="$GITHUB_WORKSPACE/sdk-source/python/src" python -m pytest contract-source/python/tests -q + GOWORK=off go -C contract-source test -count=1 ./... + - name: Build direct and sdist-derived distributions + run: | + python -m build sdk-source/python --outdir dist + mkdir derived-source derived-wheel + python - <<'PY' + import pathlib, tarfile + with tarfile.open(next(pathlib.Path('dist').glob('*.tar.gz'))) as archive: + archive.extractall('derived-source', filter='data') + PY + python -m build --wheel derived-source/* --outdir derived-wheel + - name: Bind exact source, schema and artifact identities + run: | + python scripts/check-python-release.py qualify \ + --sdk-repo sdk-source --contract-repo contract-source \ + --catalog-repo . --catalog-ref "$GITHUB_SHA" \ + --sdk-tag "$SDK_TAG" --contract-tag "$SUITE_TAG" \ + --wheel dist/*.whl --sdist dist/*.tar.gz --sdist-wheel derived-wheel/*.whl \ + --out qualification.json + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: python-release-qualified + path: | + dist/*.whl + dist/*.tar.gz + qualification.json + if-no-files-found: error + + publish: + if: github.ref == 'refs/heads/main' && inputs.publish + needs: qualify + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.14' + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: python-release-qualified + path: qualified + - name: Recheck downloaded distributions before publishing authority is used + run: python scripts/check-python-release.py verify-artifacts --qualification qualified/qualification.json --dist qualified/dist + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: qualified/dist diff --git a/docs/.vitepress/browser.test.mjs b/docs/.vitepress/browser.test.mjs index 4b84f10..659ab75 100644 --- a/docs/.vitepress/browser.test.mjs +++ b/docs/.vitepress/browser.test.mjs @@ -80,6 +80,20 @@ test('landing and specification reflow at narrow, tablet and enlarged-text width } finally { await page.close() } }) +test('Python integration is navigable and readable on desktop and mobile', async () => { + const page = await browser.newPage({ reducedMotion: 'reduce' }) + try { + for (const width of [390, 1440]) { + await page.setViewportSize({ width, height: 1000 }) + await ready(page, '/python') + assert.equal(await page.getByRole('heading', { level: 1, name: 'Python integration' }).count(), 1) + assert.equal(await page.locator('.vp-doc').getByText('Registry', { exact: false }).count() > 0, true) + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true) + assert.equal(await page.locator('.VPSidebar a[href="/python"]').count(), 1) + } + } finally { await page.close() } +}) + test('legacy specification fragments land beside their relocated sections', async () => { for (const [old, current] of [ ['822-case-documents-and-assertions', '32-case-documents-and-assertions'], diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 6c40609..b67ce7f 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -8,7 +8,7 @@ import { publishCatalog } from '../../scripts/schema-bundles.mjs' const root = fileURLToPath(new URL('../..', import.meta.url)) const repository = 'https://github.com/agent-primitives/suite' const sourceRef = process.env.GITHUB_SHA || 'main' -const pages = ['index', 'schemas', 'conformance', 'go', 'releases', 'hosting', 'legacy-draft', 'directory-identity', 'experimental-flow'] +const pages = ['index', 'schemas', 'conformance', 'go', 'python', 'releases', 'hosting', 'legacy-draft', 'directory-identity', 'experimental-flow'] export default defineConfig({ title: 'AgentSuite', @@ -72,6 +72,7 @@ export default defineConfig({ { text: 'Specification', link: '/spec' }, { text: 'Schemas', link: '/schemas' }, { text: 'Go usage', link: '/go' }, + { text: 'Python usage', link: '/python' }, { text: 'Conformance', link: '/conformance' } ], sidebar: [ @@ -80,6 +81,7 @@ export default defineConfig({ { text: 'Schemas', link: '/schemas' }, { text: 'Conformance fixtures', link: '/conformance' }, { text: 'Go usage', link: '/go' }, + { text: 'Python usage', link: '/python' }, { text: 'Versions and releases', link: '/releases' }, { text: 'Directory identity migration', link: '/directory-identity' }, { text: 'Experimental flow', link: '/experimental-flow' }, diff --git a/docs/.vitepress/site.test.mjs b/docs/.vitepress/site.test.mjs index f6275ad..e5b7c99 100644 --- a/docs/.vitepress/site.test.mjs +++ b/docs/.vitepress/site.test.mjs @@ -13,7 +13,7 @@ const html = name => readFile(path.join(dist, `${name}.html`), 'utf8') const catalog = JSON.parse(await readFile(path.join(root, 'docs/schema-catalog.json'), 'utf8')) test('all documentation pages render with root-based navigation and assets', async () => { - const pages = ['404', 'index', 'spec', 'schemas', 'conformance', 'go', 'releases', 'hosting', 'legacy-draft', 'directory-identity', 'experimental-flow'] + const pages = ['404', 'index', 'spec', 'schemas', 'conformance', 'go', 'python', 'releases', 'hosting', 'legacy-draft', 'directory-identity', 'experimental-flow'] assert.deepEqual((await readdir(dist)).filter(name => name.endsWith('.html')).sort(), pages.map(name => `${name}.html`).sort()) for (const page of pages) { const content = await html(page) @@ -26,6 +26,13 @@ test('all documentation pages render with root-based navigation and assets', asy assert.match(home, /https:\/\/github.com\/agent-primitives\/suite/) }) +test('Python integration renders the complete package-to-digest path and owner boundaries', async () => { + const content = await html('python') + for (const value of ['Package', 'Registry', 'set_instructions', 'write_oci', 'publish', 'root_digest', 'artifact_digest', 'AGENTSUITE_GO_ORACLE_ROOT']) assert.ok(content.includes(value), value) + for (const value of ['does not execute', 'origin', 'owner', 'development', 'pypi']) assert.ok(content.includes(value), value) + assert.ok((await html('go')).includes('href="/python"')) +}) + test('experimental flow documentation renders the explicit opt-in and runtime boundary', async () => { const content = await html('experimental-flow') for (const name of ['ValidatePathFlowV1Alpha1', 'ValidateDirectoryFlowV1Alpha1', 'VerifyEvaluationFlowV1Alpha1']) assert.ok(content.includes(name)) diff --git a/docs/python.md b/docs/python.md new file mode 100644 index 0000000..c39fd8f --- /dev/null +++ b/docs/python.md @@ -0,0 +1,144 @@ +# Python integration + +The `agent-suite` distribution imports as `agent_suite`. It reads, validates, +edits, writes and publishes AgentSuite packages. It does not execute agents or +evaluations, enforce policy, or qualify a runtime. + +This page tracks **main's development SDK**, currently `0.1.0.dev0`. It does not +announce a PyPI release or claim frozen Suite v0.8.0 compatibility. A release must +pass the frozen-contract qualification described below before publication. + +## Install from a checkout + +From the repository root, with CPython 3.10 or newer: + +```sh +python -m pip install --only-binary=google-re2 ./python +``` + +The native RE2 dependency is exactly pinned; unsupported platforms must not fall +back to a source build or Python `re`. The CI matrix covers Python 3.10 and 3.14 +on Linux, macOS and Windows. Go is a conformance reference, not a runtime dependency. +Schemas ship as exact, offline package assets; loading never downloads schemas. + +## Inspect, edit and publish + +Use an extracted content directory or local OCI layout. The selected package must +contain the `writer` agent used here. Both output paths must be fresh, and their +existing parents must satisfy the native reader's ownership and permission checks. + +```python +import os +from agent_suite import Package, Registry + +credentials = { + "https://registry.example": ( + os.environ["REGISTRY_USERNAME"], + os.environ["REGISTRY_PASSWORD"], + ), +} + +with Package.load("./support-suite") as package: + agent = package.agents["writer"] + print(agent.declaration.id, agent.source.file, agent.source.pointer) + instructions = "".join(agent.instructions.content.iter_text()) + package.set_instructions("writer", instructions + "\nAnswer briefly.\n") + assert package.validate() == () + layout = package.write_oci("./out-layout") + +reference = Registry("registry.example", credentials=credentials).publish( + layout, "team/support" +) +print(reference) # registry.example/team/support@sha256:... +``` + +`Package.load` performs complete schema and graph validation, including policy and +evaluation declarations. `validate()` checks the captured package, not a running +agent. Raw JSON mutation is unsupported; `set_instructions` refreshes the selected +literal, agent and Suite pins bottom-up and refuses a stale input graph. + +After an edit, reacquire views: earlier frozen metadata remains historical, but +old literal handles are closed. To read authored JSON metadata without inventing +defaults, use `json.loads(agent.document.data)`; typed declarations and literal +content remain separate. `iter_text()` decodes UTF-8 strictly; `iter_bytes()` +preserves arbitrary literal bytes. + +## Keep the identities separate + +- `root_digest` is the validated Suite root's RFC 8785 identity. It is not an OCI + address or an aggregate hash of every content file. +- `digest(value)` and `canonical_bytes(value)` use RFC 8785. `variant_digest(value)` + blanks only the top-level `variantDigest`. Instruction and schema literals hash + their exact bytes rather than canonicalized JSON. +- `artifact_digest` identifies the **exact OCI manifest bytes**. It is absent for + extracted directories and cleared by edits; reload the written OCI layout for + its new identity. `oci_manifest_digest(manifest_bytes)` hashes those exact bytes. +- `Registry.publish` returns `authority/repository@sha256:…` using that exact + manifest identity, never `root_digest`. + +Unedited OCI copies preserve their original envelope, manifest and compressed +layer. Regenerated OCI output is deterministic. Native directory writing may +refuse modes accepted in OCI; writers do not repair modes or flatten unsupported +links. No public aggregate-content identity or generic manifest builder is exposed. + +## Credentials, transfer and failures + +`Registry(origin, *, credentials=None, timeout=30.0)` accepts a registry authority +(defaulting to HTTPS) or an explicit HTTP(S) origin. Explicit HTTP is unencrypted; +use it only where that is intended, such as an owned loopback test registry. +Repository names are passed separately to `publish(layout, repository)`; this +method accepts a fully validated local OCI layout, not an extracted directory. + +Credentials are a mapping from **exact normalized HTTP origins** to +`(username, password)` tuples. A registry token can be the password. If a Bearer +challenge uses a different token-service origin, include credentials for that +origin explicitly when it needs them; registry credentials are not copied there. +No credential files, netrc, ambient proxy credentials or response cookies are used. +The client does not expose a credential store, login flow or transport framework. + +All transfer input is verified and captured into bounded-memory, seekable disk +snapshots before the first request. The source may then disappear without +changing the uploaded bytes. Config and layer blobs are uploaded before the +original manifest, addressed by digest. A final manifest read confirms exact +remote bytes before the reference is returned; it is verification of this upload, +not a registry pull API. Already-present blobs make repeated publication cheap. + +Server upload locations retain their existing query parameters. Authorization is +not forwarded to a different upload origin. Bounded 307/308 redirects preserve +request methods and replay the snapshot; HTTPS is not downgraded. Other redirect +statuses and unsupported responses fail rather than changing a PUT into a GET. +No multipart, chunk-resume, background retry or registry cleanup framework exists. + +Local `LoadError` refusals retain file/pointer diagnostics. Transfer failures raise +contextual `ValueError` without server bodies, secret-bearing URLs or tokens. +Failure after a remote write can leave blobs or a manifest present; it never +returns a guessed success. Retry the whole publication to confirm the same digest. + +## Owner-controlled release qualification + +The manual `python-release.yml` workflow takes an existing `python/vX.Y.Z` SDK tag, +a frozen `v0.8.P` Suite tag, and `publish` (default **false**). It never creates tags +or changes schema IDs. The first release requires the final SDK code and version +bump in the frozen v0.8 release commit; tag that immutable source, merge the +reviewed schema-catalog update, then dispatch qualification from main. Main can +restore draft IDs afterwards because qualification checks out both exact tags. +Frozen maintenance-branch SDK patch tags are supported without requiring ancestry +from current draft main. + +Trusted tools and the reviewed catalog are pinned to the dispatch revision. They +check exact schema bytes, source/tag/version identities, wheel and sdist provenance, +and Go conformance from the selected contract revision. Test-only +`AGENTSUITE_GO_ORACLE_ROOT` selects that frozen Go checkout; it adds no runtime Go. +The downloaded wheel and sdist are re-hashed against the qualification record +before any upload. + +The release owner must configure PyPI Trusted Publishing and a protected `pypi` +environment requiring that owner's approval before opting into upload. Running +qualification does not upload by default. The first PyPI upload and Suite v0.8.0 +release are owner actions, not actions performed by a pull request. + +## Boundaries + +The SDK does not run tests, models, providers or deployment tools. Stored test and +policy validation is not execution or enforcement. There is no registry pull, +tag publication, signing, referrer, CLI, runtime wrapper or telemetry API. diff --git a/docs/releases.md b/docs/releases.md index 5e83128..5f37aab 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -371,6 +371,13 @@ execution contracts. v0.2.0 introduced the filesystem policy package. See the [normative policy contract](../spec.md#appendix-b-standalone-policy-contracts-normative) and [reference schemas](schemas.md). +## Python release preparation + +Before the first SDK release, the Suite v0.8 freeze commit must also contain the +final SDK code and SDK version bump. Keep those immutable tags, merge the reviewed +schema catalog, then follow [Python release qualification](python.md#owner-controlled-release-qualification). +Qualification is not an upload; the protected `pypi` environment requires release-owner approval. + ## Compatibility Go module versions and format/schema versions are independent. A Go release does diff --git a/python/README.md b/python/README.md index 9292757..e654989 100644 --- a/python/README.md +++ b/python/README.md @@ -7,9 +7,10 @@ bytes. It does not execute evaluations, qualify a runtime, enforce policy or resolve administrative restrictions. This development distribution is not a PyPI release. Publication and release -qualification are separate maintainer steps. Local writing and canonical identity -helpers are included; registry clients, network fetching and public outer-archive -loading are not. +qualification are separate maintainer steps. Local writing, canonical identity +helpers and digest-preserving registry publication are included; registry pull and +public outer-archive loading are not. See the [complete integration path](../docs/python.md) +for explicit credentials, publication failures and owner-controlled release plumbing. ## Install @@ -89,7 +90,8 @@ Instruction and tool-schema pins hash their exact literal bytes. `artifact_digest` is the distinct SHA-256 of the exact OCI manifest bytes, or `None` for directory input. It is not the Suite root digest or an aggregate hash -of content. No public aggregate digest or registry-reference API is provided. +of content. No public aggregate digest is provided. `Registry.publish` returns a registry reference +using the exact OCI manifest identity, not this root/document identity. The pure `canonical_bytes(value)` and `digest(value)` helpers use RFC 8785, not sorted Python JSON. `variant_digest(value)` blanks only the top-level diff --git a/python/pyproject.toml b/python/pyproject.toml index 07e236b..ecb5eac 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -9,7 +9,7 @@ description = "AgentSuite package reader and format models" readme = "README.md" requires-python = ">=3.10" license = "MIT" -dependencies = ["rfc8785==0.1.4", "jsonschema==4.26.0", "referencing==0.37.0", "google-re2==1.1.20251105"] +dependencies = ["rfc8785==0.1.4", "jsonschema==4.26.0", "referencing==0.37.0", "google-re2==1.1.20251105", "requests==2.34.2"] [tool.setuptools.packages.find] where = ["src"] diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index b30a4de..66fce62 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -1,4 +1,5 @@ build==1.6.1 mypy==2.4.0 pytest==9.1.1 +types-requests==2.33.0.20261006 setuptools==84.2.1 diff --git a/python/src/agent_suite/__init__.py b/python/src/agent_suite/__init__.py index d86c1cb..cf9f6aa 100644 --- a/python/src/agent_suite/__init__.py +++ b/python/src/agent_suite/__init__.py @@ -2,6 +2,7 @@ from ._identity import digest, oci_manifest_digest, variant_digest from ._json import Number, canonical_bytes from ._literals import LiteralContent +from ._registry import Registry from ._models import Descriptor, Diagnostic, Document, Platform, Source from ._package import ( AgentView, CaseView, EvaluationIndexView, LiteralView, LoadError, Package, @@ -22,7 +23,7 @@ __all__ = [ 'canonical_bytes', 'digest', 'variant_digest', 'oci_manifest_digest', - 'Package', 'LoadError', 'Source', 'Diagnostic', 'Document', 'Descriptor', 'Platform', 'Number', + 'Registry', 'Package', 'LoadError', 'Source', 'Diagnostic', 'Document', 'Descriptor', 'Platform', 'Number', 'AgentView', 'CaseView', 'EvaluationIndexView', 'LiteralView', 'LiteralContent', 'PolicyView', 'ProviderView', 'ToolView', 'Agent', 'AgentInvoke', 'BrokeredCredential', 'BuildProfile', 'Catalog', 'Composition', 'CompositionRef', 'DelegatedAuthorization', 'EnvironmentName', 'EvaluationAssertion', diff --git a/python/src/agent_suite/_registry.py b/python/src/agent_suite/_registry.py new file mode 100644 index 0000000..56034aa --- /dev/null +++ b/python/src/agent_suite/_registry.py @@ -0,0 +1,114 @@ +"""Digest-preserving publication of validated local OCI layouts.""" +from __future__ import annotations + +import hashlib +import os +import re +import tempfile +from collections.abc import Mapping +from contextlib import ExitStack +from pathlib import Path +from typing import BinaryIO, cast +from urllib.parse import parse_qs, quote, urlsplit, urlunsplit + +import requests + +from . import _oci_json as j +from ._json import MAX_BYTES, digest_bytes +from ._oci import _open_blob, _read_blob +from ._oci_io import read_regular +from ._package import Package +from ._registry_http import HTTP, location, origin as http_origin, read_response, require + +_NAME = re.compile(r'[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*') + + +class Registry: + """Publish exact OCI bytes; credentials are explicitly keyed by HTTP origin.""" + + def __init__(self, origin: str, *, credentials: Mapping[str, tuple[str, str]] | None = None, + timeout: float = 30.0): + raw = origin if '://' in origin else 'https://' + origin + self._origin = http_origin(raw) + parsed = urlsplit(raw) + if parsed.path not in ('', '/') or parsed.query: + raise ValueError('registry address must name an origin') + self._credentials = {} + for key, pair in (credentials or {}).items(): + target = http_origin(key) + if key.rstrip('/') != target or not isinstance(pair, tuple) or len(pair) != 2 or not all(isinstance(v, str) for v in pair): + raise ValueError('registry credentials must be origin-keyed username/password pairs') + self._credentials[target] = pair + self._timeout = timeout + + def publish(self, layout: str | os.PathLike[str], repository: str) -> str: + """Validate and snapshot before HTTP; return only a remotely verified reference.""" + if not isinstance(repository, str) or len(repository) > 255 or not _NAME.fullmatch(repository): + raise ValueError('invalid OCI repository name') + with ExitStack() as stack: + blobs: list[tuple[j.Descriptor, BinaryIO]] = [] + root = Path(layout) + with Package.load(root) as package: + if package._content.source_type != 'oci-layout': + raise ValueError('registry publication requires a local OCI layout') + selected = j.index(read_regular(root/'index.json', MAX_BYTES)) + if len(selected) != 1 or selected[0].digest != package.artifact_digest: + raise ValueError('OCI artifact changed before publication') + manifest = _read_blob(root, selected[0]) + config, layer = j.manifest(manifest) + for descriptor in (config, layer): + snapshot = cast(BinaryIO, stack.enter_context(tempfile.TemporaryFile())) + hasher = hashlib.sha256() + size = 0 + with _open_blob(root, descriptor) as source: + while chunk := source.read(128 << 10): + snapshot.write(chunk) + hasher.update(chunk) + size += len(chunk) + if size > descriptor.size: + raise ValueError('OCI blob changed during publication snapshot') + if size != descriptor.size or 'sha256:' + hasher.hexdigest() != descriptor.digest: + raise ValueError('OCI blob changed during publication snapshot') + snapshot.seek(0) + blobs.append((descriptor, snapshot)) + http = HTTP(self._origin, self._credentials, self._timeout) + stack.callback(http.close) + base = self._origin + '/v2/' + repository + try: + for descriptor, snapshot in blobs: + with http.request('HEAD', base + '/blobs/' + descriptor.digest, 'blob check') as response: + if response.status_code == 200: + require(response, 200, 'blob check', descriptor.digest) + size_header = response.headers.get('Content-Length') + if size_header is not None and size_header != str(descriptor.size): + raise ValueError('registry blob size mismatch') + continue + require(response, 404, 'blob check') + with http.request('POST', base + '/blobs/uploads/', 'upload start') as response: + require(response, 202, 'upload start') + value = response.headers.get('Location', '') + if not value: + raise ValueError('registry upload location missing') + target = location(response.url, value) + parsed = urlsplit(target) + query = parse_qs(parsed.query) + if 'digest' in query and query['digest'] != [descriptor.digest]: + raise ValueError('registry upload location digest mismatch') + if 'digest' not in query: + target = urlunsplit(parsed._replace(query=parsed.query + ('&' if parsed.query else '') + 'digest=' + quote(descriptor.digest, safe=''))) + fields = {'Content-Type': 'application/octet-stream', 'Content-Length': str(descriptor.size)} + with http.request('PUT', target, 'blob upload', body=snapshot, headers=fields) as response: + require(response, 201, 'blob upload', descriptor.digest) + digest = digest_bytes(manifest) + target = base + '/manifests/' + digest + with http.request('PUT', target, 'manifest upload', body=manifest, + headers={'Content-Type': j.MANIFEST}) as response: + require(response, 201, 'manifest upload', digest) + with http.request('GET', target, 'manifest verification', + headers={'Accept': j.MANIFEST, 'Accept-Encoding': 'identity'}) as response: + require(response, 200, 'manifest verification', digest) + if read_response(response) != manifest: + raise ValueError('registry manifest verification digest mismatch') + except (requests.RequestException, UnicodeError): + raise ValueError('registry publication request failed') from None + return urlsplit(self._origin).netloc + '/' + repository + '@' + digest diff --git a/python/src/agent_suite/_registry_http.py b/python/src/agent_suite/_registry_http.py new file mode 100644 index 0000000..e02e618 --- /dev/null +++ b/python/src/agent_suite/_registry_http.py @@ -0,0 +1,170 @@ +"""Private request/auth handling for one digest publication, not a general client.""" +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import BinaryIO +from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit +from urllib.request import parse_http_list, parse_keqv_list + +import requests +from requests.auth import _basic_auth_str + +from ._json import MAX_BYTES + + +def origin(url: str) -> str: + try: + parsed = urlsplit(url) + host = parsed.hostname + port = parsed.port + if (parsed.scheme not in ('http', 'https') or not host or parsed.username is not None + or parsed.password is not None or parsed.fragment or any(ord(c) <= 32 for c in url)): + raise ValueError + # Bind credentials to the same IDNA/URL authority Requests will send. + prepared = requests.PreparedRequest() + prepared.prepare_url(url, None) + if not isinstance(prepared.url, str): + raise ValueError + host = urlsplit(prepared.url).hostname + if not host: + raise ValueError + authority = '[' + host + ']' if ':' in host else host + if port is not None: + if not 1 <= port <= 65535: + raise ValueError + if port != (443 if parsed.scheme == 'https' else 80): + authority += ':' + str(port) + return parsed.scheme + '://' + authority + except (ValueError, UnicodeError, AttributeError, requests.RequestException): + raise ValueError('invalid registry HTTP origin or location') from None + + +def location(base: str, value: str) -> str: + target = urljoin(base, value) + target_origin = origin(target) + if urlsplit(base).scheme == 'https' and urlsplit(target_origin).scheme != 'https': + raise ValueError('registry location downgrades HTTPS') + return target + + +def read_response(response: requests.Response) -> bytes: + data = bytearray() + for chunk in response.iter_content(128 << 10): + data.extend(chunk) + if len(data) > MAX_BYTES: + raise ValueError('registry response exceeds JSON byte limit') + return bytes(data) + + +class HTTP: + def __init__(self, base: str, credentials: Mapping[str, tuple[str, str]], timeout: float): + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError('registry timeout must be positive and finite') + self.base = base + self.timeout = timeout + self.credentials = credentials + self.authorization = '' + self.session = requests.Session() + self.session.trust_env = False + + def close(self) -> None: + self.session.close() + + def request(self, method: str, url: str, stage: str, *, body: BinaryIO | bytes | None = None, + headers: Mapping[str, str] | None = None, authenticate: bool = True) -> requests.Response: + current = url + fields = dict(headers or {}) + initial_origin = origin(url) + supplied = fields.pop('Authorization', '') + negotiated = False + redirects = 0 + while True: + current_origin = origin(current) + fields.pop('Authorization', None) + if authenticate and current_origin == self.base and self.authorization: + fields['Authorization'] = self.authorization + elif supplied and current_origin == initial_origin: + fields['Authorization'] = supplied + if body is not None and not isinstance(body, bytes): + body.seek(0) + self.session.cookies.clear() + try: + response = self.session.request(method, current, data=body, headers=fields, + allow_redirects=False, timeout=self.timeout, stream=True) + except requests.RequestException: + raise ValueError('registry ' + stage + ' request failed') from None + if response.status_code == 401 and authenticate and current_origin == self.base and not negotiated: + challenge = response.headers.get('WWW-Authenticate', '') + response.close() + self.authorization = self._authenticate(challenge) + negotiated = True + continue + if response.status_code in (307, 308): + value = self.session.get_redirect_target(response) + response.close() + if not value or redirects == 3: + raise ValueError('registry ' + stage + ' redirect failed') from None + current = location(current, value) + if origin(current) != current_origin: + supplied = '' + redirects += 1 + continue + return response + + def _authenticate(self, challenge: str) -> str: + challenges: list[tuple[str, list[str]]] = [] + for item in parse_http_list(challenge): + parts = item.split(None, 1) + if len(parts) == 2 and '=' not in parts[0] and not parts[1].lstrip().startswith('='): + challenges.append((parts[0].lower(), [parts[1]])) + elif challenges: + challenges[-1][1].append(item) + else: + raise ValueError('invalid registry authentication challenge') from None + scheme, parameters = next((item for item in challenges if item[0] == 'bearer'), + challenges[0] if challenges else ('', [])) + if scheme == 'basic': + pair = self.credentials.get(self.base) + if pair is None: + raise ValueError('registry credentials required') from None + return _basic_auth_str(*pair) + if scheme != 'bearer': + raise ValueError('registry authentication challenge is unsupported') from None + fields = {} + for item in parameters: + key, separator, value = item.partition('=') + key = key.strip().lower() + if not separator or key in fields: + raise ValueError('invalid registry Bearer challenge') from None + parsed = parse_keqv_list([key + '=' + value.strip()]) + fields[key] = next(iter(parsed.values())) + realm = fields.get('realm', '') + if not realm: + raise ValueError('registry Bearer realm missing') from None + target = location(self.base + '/', realm) + parsed_url = urlsplit(target) + query = urlencode([(key, fields[key]) for key in ('service', 'scope') if key in fields]) + target = urlunsplit(parsed_url._replace(query=parsed_url.query + ('&' if parsed_url.query and query else '') + query)) + headers = {} + pair = self.credentials.get(origin(target)) + if pair is not None: + headers['Authorization'] = _basic_auth_str(*pair) + with self.request('GET', target, 'token', headers=headers, authenticate=False) as response: + require(response, 200, 'token') + import json + try: + document = json.loads(read_response(response)) + token = document.get('token') or document.get('access_token') + if not isinstance(token, str) or not token or any(ord(c) <= 32 for c in token): + raise ValueError + except (ValueError, AttributeError, TypeError): + raise ValueError('invalid registry token response') from None + return 'Bearer ' + token + + +def require(response: requests.Response, status: int, stage: str, digest: str = '') -> None: + if response.status_code != status: + raise ValueError(f'registry {stage} failed (HTTP {response.status_code})') from None + if digest and response.headers.get('Docker-Content-Digest', digest) != digest: + raise ValueError('registry ' + stage + ' digest mismatch') from None diff --git a/python/tests/_registry_fixture.py b/python/tests/_registry_fixture.py new file mode 100644 index 0000000..3402056 --- /dev/null +++ b/python/tests/_registry_fixture.py @@ -0,0 +1,98 @@ +"""Owned loopback registry for actual HTTP requests, never a provider service.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit + + +class LocalRegistry: + def __init__(self): + self.blobs = {} + self.manifests = {} + self.calls = [] + self.override = lambda method, path, headers, body: None + self.auth = None + self.realm = None + self.location = None + self.digest_headers = True + self.password = 'fixture-password-not-a-credential' + self.token = 'fixture-bearer-not-a-credential' + self.base = '' + + def reply(self, method, path, headers, body): + self.calls.append((method, path, dict(headers), body)) + override = self.override(method, path, headers, body) + if override is not None: + return override + parts = urlsplit(path) + basic = 'Basic ' + base64.b64encode(('fixture:' + self.password).encode()).decode() + if parts.path == '/token': + if self.auth == 'basic' and headers.get('Authorization') != basic: + return 401, {}, b'' + return 200, {'Content-Type': 'application/json'}, json.dumps({'token': self.token}).encode() + if self.auth and headers.get('Authorization') != (basic if self.auth == 'basic' else 'Bearer ' + self.token): + challenge = 'Basic realm="fixture"' if self.auth == 'basic' else ( + 'Bearer realm="' + (self.realm or self.base + '/token') + '",service="fixture",scope="repository:team/suite:pull,push"') + return 401, {'WWW-Authenticate': challenge}, b'' + if method == 'POST' and parts.path.endswith('/blobs/uploads/'): + return 202, {'Location': self.location or parts.path + 'fixture?state=keep%2Bexact'}, b'' + if method == 'PUT' and '/blobs/uploads/' in parts.path: + digest = parse_qs(parts.query).get('digest', [''])[0] + if digest != 'sha256:' + hashlib.sha256(body).hexdigest(): + return 400, {}, b'' + self.blobs[digest] = body + return 201, {'Docker-Content-Digest': digest} if self.digest_headers else {}, b'' + if '/blobs/' in parts.path and method == 'HEAD': + digest = parts.path.rsplit('/', 1)[1] + if digest not in self.blobs: + return 404, {}, b'' + return 200, {'Content-Length': str(len(self.blobs[digest]))}, b'' + if '/manifests/' in parts.path: + digest = parts.path.rsplit('/', 1)[1] + if method == 'PUT': + if digest != 'sha256:' + hashlib.sha256(body).hexdigest(): + return 400, {}, b'' + self.manifests[digest] = body + return 201, {'Docker-Content-Digest': digest} if self.digest_headers else {}, b'' + if method == 'GET' and digest in self.manifests: + return 200, {'Content-Type': 'application/vnd.oci.image.manifest.v1+json'}, self.manifests[digest] + return 404, {}, b'' + + +@contextmanager +def registry_server(): + registry = LocalRegistry() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def dispatch(self): + body = self.rfile.read(int(self.headers.get('Content-Length', '0'))) + status, headers, output = registry.reply(self.command, self.path, self.headers, body) + self.send_response(status) + for key, value in headers.items(): + self.send_header(key, value) + if 'Content-Length' not in headers: + self.send_header('Content-Length', str(len(output))) + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(output) + + do_GET = do_HEAD = do_POST = do_PUT = dispatch + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + registry.base = f'http://127.0.0.1:{server.server_port}' + thread.start() + try: + yield registry + finally: + server.shutdown() + server.server_close() + thread.join() diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 0f43fa0..c8c0b16 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -191,6 +191,83 @@ 'test_writer_retains_output_descriptors_until_completion', 'test_writer_readonly_inventory_directory_with_link', } +REQUIRED |= { + "test_registry_auth_replay_rewinds_body", + "test_registry_basic_and_bearer_credentials_are_origin_bound[basic]", + "test_registry_basic_and_bearer_credentials_are_origin_bound[bearer]", + "test_registry_bearer_realm_does_not_inherit_registry_password", + "test_registry_does_not_discover_netrc_or_cookies", + "test_registry_errors_and_repr_redact_secrets", + "test_registry_invalid_origins_refuse[]", + "test_registry_invalid_origins_refuse[file:///path]", + "test_registry_invalid_origins_refuse[https://fixture:password@registry.test]", + "test_registry_invalid_origins_refuse[https://registry.test/path]", + "test_registry_invalid_origins_refuse[https://registry.test?secret=query]", + "test_registry_malformed_challenges_refuse_safely[Bearer realm=\"ftp://invalid.test/token\"]", + "test_registry_malformed_challenges_refuse_safely[Bearer realm=\"{base}/token\",broken]", + "test_registry_malformed_challenges_refuse_safely[Bearer realm=\"{base}/token\",realm=\"{base}/other\"]", + "test_registry_malformed_challenges_refuse_safely[Bearer service=\"fixture\"]", + "test_registry_malformed_challenges_refuse_safely[Unknown realm=\"fixture\"]", + "test_registry_malformed_tokens_refuse_safely[[]]", + "test_registry_malformed_tokens_refuse_safely[not-json]", + "test_registry_malformed_tokens_refuse_safely[{\"token\":\"bad token\"}]", + "test_registry_malformed_tokens_refuse_safely[{\"token\":null}]", + "test_registry_malformed_tokens_refuse_safely[{}]", + "test_registry_publish_preserves_manifest_and_blobs", + "test_registry_refuses_before_network_on_invalid_or_changed_layout[corrupt]", + "test_registry_refuses_before_network_on_invalid_or_changed_layout[directory]", + "test_registry_releases_streams_and_session[False]", + "test_registry_releases_streams_and_session[True]", + "test_registry_repeat_push_and_missing_optional_headers", + "test_registry_snapshot_rechecks_bytes_before_http", + "test_registry_snapshots_outlive_source", + "test_registry_uncertain_or_mismatched_remote_never_returns_reference[digest-header]", + "test_registry_uncertain_or_mismatched_remote_never_returns_reference[final-response]", + "test_registry_uncertain_or_mismatched_remote_never_returns_reference[head-size]", + "test_registry_uncertain_or_mismatched_remote_never_returns_reference[manifest-digest]", + "test_registry_upload_queries_and_redirects_preserve_bytes", + "test_registry_validates_target_and_closes_resources[../suite]", + "test_registry_validates_target_and_closes_resources[Team/suite]", + "test_registry_validates_target_and_closes_resources[]", + "test_registry_validates_target_and_closes_resources[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]", + "test_registry_validates_target_and_closes_resources[team//suite]", + "test_registry_validates_target_and_closes_resources[team/suite:tag]", +} +REQUIRED.add('test_registry_selects_bearer_from_multiple_challenges') +REQUIRED |= { + "test_python_release_accepts_frozen_maintenance_tags", + "test_python_release_binds_exact_frozen_contract_and_schema_bytes[catalog]", + "test_python_release_binds_exact_frozen_contract_and_schema_bytes[draft]", + "test_python_release_binds_exact_frozen_contract_and_schema_bytes[oracle]", + "test_python_release_binds_exact_frozen_contract_and_schema_bytes[schema]", + "test_python_release_checks_both_distribution_provenances", + "test_python_release_oracle_override_selects_build_and_run", + "test_python_release_records_validator_and_artifact_identities", + "test_python_release_rejects_development_or_wrong_tags[contract-tag]", + "test_python_release_rejects_development_or_wrong_tags[development-tag]", + "test_python_release_rejects_development_or_wrong_tags[development]", + "test_python_release_rejects_development_or_wrong_tags[retargeted-contract]", + "test_python_release_rejects_development_or_wrong_tags[retargeted-sdk]", + "test_python_release_rejects_development_or_wrong_tags[sdk-tag]", + "test_python_release_rejects_downloaded_artifact_changes[extra]", + "test_python_release_rejects_downloaded_artifact_changes[missing]", + "test_python_release_rejects_downloaded_artifact_changes[tampered]", +} +REQUIRED |= { + 'test_registry_idna_transport_origin_binds_credentials[False]', + 'test_registry_idna_transport_origin_binds_credentials[True]', + 'test_registry_unicode_auth_failures_are_redacted[basic]', + 'test_registry_unicode_auth_failures_are_redacted[realm-basic]', + 'test_registry_unicode_auth_failures_are_redacted[bearer-token]', + 'test_registry_bearer_parameters_allow_optional_whitespace', + 'test_python_release_all_differentials_use_selected_contract[test_codec-test_go_numeric_differential-False-False]', + 'test_python_release_all_differentials_use_selected_contract[test_directory-test_go_casefold_differential-False-False]', + 'test_python_release_all_differentials_use_selected_contract[test_archive_differential-test_go_archive_differential-True-True]', + 'test_python_release_all_differentials_use_selected_contract[test_oci_differential-test_go_oci_differential-True-True]', + 'test_python_release_all_differentials_use_selected_contract[test_regex-test_go_regex_differential-True-True]', + 'test_python_release_qualifies_frozen_vector_suite', + 'test_python_release_unmapped_maintenance_tag_refused', +} PASSED = set() SKIPPED = [] diff --git a/python/tests/test_archive_differential.py b/python/tests/test_archive_differential.py index 3d312fb..29545c8 100644 --- a/python/tests/test_archive_differential.py +++ b/python/tests/test_archive_differential.py @@ -9,6 +9,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() def header(name="file.json", kind=b"0", size=0): @@ -99,8 +100,8 @@ def add(name, raw): requests = tmp_path / "requests.json" requests.write_text(json.dumps(probes), encoding="utf-8") executable = tmp_path / ("oracle.exe" if os.name == "nt" else "oracle") - subprocess.run(["go", "test", "-c", "-o", str(executable), "./internal/artifact"], cwd=ROOT, check=True) - subprocess.run([str(executable), "-test.run=^TestSharedArchiveCases$"], cwd=ROOT, check=True, + subprocess.run(["go", "test", "-c", "-o", str(executable), "./internal/artifact"], cwd=ORACLE_ROOT, check=True) + subprocess.run([str(executable), "-test.run=^TestSharedArchiveCases$"], cwd=ORACLE_ROOT, check=True, env={**os.environ, "AGENTSUITE_ARCHIVE_ORACLE": str(requests)}, capture_output=True) expected = json.loads(Path(str(requests) + ".result").read_bytes()) mismatches = [] diff --git a/python/tests/test_build.py b/python/tests/test_build.py index a73bc30..7218b47 100644 --- a/python/tests/test_build.py +++ b/python/tests/test_build.py @@ -124,7 +124,8 @@ def counted_equal(left, right): # Both distribution paths exercise the public complete reader with the REAL # native backend, full authored evaluations and no runtime checkout dependency. import shutil -from agent_suite import Package, CaseView, PolicyV2 +from agent_suite import Package, CaseView, PolicyV2, Registry +import sys public_directory = Package.load('rich-directory') public_oci = Package.load('rich-layout') assert public_directory.root_digest == public_oci.root_digest @@ -154,6 +155,8 @@ def counted_equal(left, right): import json selected = json.loads((path/'index.json').read_bytes())['manifests'][0]['digest'] assert oci_manifest_digest((path/'blobs/sha256'/selected[7:]).read_bytes()) == output.artifact_digest + reference = Registry(sys.argv[1]).publish(path, 'installed/suite') + assert reference.endswith('@' + output.artifact_digest) shutil.rmtree('rich-directory') shutil.rmtree('rich-layout') for package in (public_directory,public_oci): @@ -235,5 +238,9 @@ def counted_equal(left, right): output_parent.mkdir(exist_ok=True) if os.name == 'nt': secure_fixture(output_parent) - subprocess.run([str(python), "-I", "-c", smoke], cwd=tmp_path, check=True) + from _registry_fixture import registry_server + with registry_server() as remote: + subprocess.run([str(python), "-I", "-c", smoke, remote.base], cwd=tmp_path, check=True) + assert remote.manifests and len(remote.blobs) >= 2 + assert sum(method == 'PUT' and '/manifests/' in path for method, path, _, _ in remote.calls) == 2 shutil.rmtree(output_parent) diff --git a/python/tests/test_codec.py b/python/tests/test_codec.py index 72c2b81..a75b429 100644 --- a/python/tests/test_codec.py +++ b/python/tests/test_codec.py @@ -2,11 +2,13 @@ import hashlib import importlib.util import json +import os from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() CORPUS = ROOT / "testdata/conformance" @@ -158,7 +160,7 @@ def test_go_numeric_differential(): requests += [{"Input": '{"mediaType":"x","digest":"x","size":' + number + '}', "Target": "descriptor"} for number in lexemes] oracle = subprocess.run( - ["go", "run", "./python/tests/testdata/oracle.go"], cwd=ROOT, + ["go", "run", "./python/tests/testdata/oracle.go"], cwd=ORACLE_ROOT, input="".join(json.dumps(item) + "\n" for item in requests), text=True, capture_output=True, check=True, ) diff --git a/python/tests/test_directory.py b/python/tests/test_directory.py index 2fbd391..cb5d2b5 100644 --- a/python/tests/test_directory.py +++ b/python/tests/test_directory.py @@ -9,6 +9,7 @@ import pytest ROOT = Path(__file__).resolve().parents[2] +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() def named_rows(data, key, minimum, name="name"): @@ -260,8 +261,8 @@ def test_go_casefold_differential(): # A new Unicode endpoint needs full-scalar verification, not a runtime refusal. assert unicodedata.unidata_version in {"13.0.0", "14.0.0", "15.0.0", "15.1.0", "16.0.0"} result = subprocess.run( - ["go", "run", str(ROOT / "python/tests/testdata/fold_oracle.go")], - cwd=ROOT, check=True, capture_output=True, + ["go", "run", str(ORACLE_ROOT / "python/tests/testdata/fold_oracle.go")], + cwd=ORACLE_ROOT, check=True, capture_output=True, ) reference = json.loads(result.stdout) for scalar in range(0x110000): diff --git a/python/tests/test_oci_differential.py b/python/tests/test_oci_differential.py index 47f6bda..dfa8738 100644 --- a/python/tests/test_oci_differential.py +++ b/python/tests/test_oci_differential.py @@ -9,6 +9,8 @@ from test_oci import ROOT, materialize, snapshot +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() + def test_go_oci_differential(tmp_path): from agent_suite._oci import load_oci_layout @@ -62,8 +64,8 @@ def add(manifest, idx=None): requests = tmp_path / 'requests.json' requests.write_text(json.dumps(probes), encoding='utf-8') executable = tmp_path / ('oracle.exe' if os.name == 'nt' else 'oracle') - subprocess.run(['go', 'test', '-c', '-o', str(executable), './internal/artifact'], cwd=ROOT, check=True) - subprocess.run([str(executable), '-test.run=^TestSharedOCICases$'], cwd=ROOT / 'internal/artifact', check=True, + subprocess.run(['go', 'test', '-c', '-o', str(executable), './internal/artifact'], cwd=ORACLE_ROOT, check=True) + subprocess.run([str(executable), '-test.run=^TestSharedOCICases$'], cwd=ORACLE_ROOT / 'internal/artifact', check=True, env={**os.environ, 'AGENTSUITE_OCI_ORACLE': str(requests)}, capture_output=True, timeout=60) expected = json.loads(Path(str(requests) + '.result').read_bytes()) mismatches = [] diff --git a/python/tests/test_regex.py b/python/tests/test_regex.py index 135d60e..1a70a5e 100644 --- a/python/tests/test_regex.py +++ b/python/tests/test_regex.py @@ -10,6 +10,7 @@ import re2 ROOT = Path(__file__).resolve().parents[2] +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() def _corpus(): @@ -225,9 +226,9 @@ def test_go_regex_differential(tmp_path): request.write_text(json.dumps(patterns), encoding="utf-8") executable = tmp_path / ("oracle.exe" if os.name == "nt" else "oracle") subprocess.run(["go", "test", "-c", "-o", str(executable), "./internal/artifact"], - cwd=ROOT, check=True, capture_output=True, timeout=120) + cwd=ORACLE_ROOT, check=True, capture_output=True, timeout=120) subprocess.run([str(executable), "-test.run=^TestRegexDifferentialOracle$"], - cwd=ROOT / "internal/artifact", check=True, capture_output=True, + cwd=ORACLE_ROOT / "internal/artifact", check=True, capture_output=True, env={**os.environ, "AGENTSUITE_REGEX_ORACLE": str(request)}, timeout=120) expected = json.loads(Path(str(request) + ".result").read_bytes()) mismatches = [(pattern, want, valid(pattern)) diff --git a/python/tests/test_registry.py b/python/tests/test_registry.py new file mode 100644 index 0000000..e54b1f3 --- /dev/null +++ b/python/tests/test_registry.py @@ -0,0 +1,181 @@ +import hashlib +import json +from urllib.parse import urlsplit + +import pytest +import agent_suite as sdk + +from _registry_fixture import registry_server +from test_capture import extracted, secure +from test_package import api + + +@pytest.fixture +def layout(tmp_path, api): + secure(tmp_path) + root, _ = extracted(tmp_path) + secure(root) + with api.Package.load(root) as package: + return package.write_oci(tmp_path/'layout') + + +def test_registry_publish_preserves_manifest_and_blobs(layout, api): + manifest = json.loads((layout/'index.json').read_bytes())['manifests'][0] + digest = manifest['digest'] + raw = (layout/'blobs/sha256'/digest[7:]).read_bytes() + document = json.loads(raw) + with registry_server() as remote: + reference = sdk.Registry(origin=remote.base).publish(layout, 'team/suite') + assert reference == urlsplit(remote.base).netloc + '/team/suite@' + digest + assert remote.manifests[digest] == raw + for item in [document['config'], *document['layers']]: + assert remote.blobs[item['digest']] == (layout/'blobs/sha256'/item['digest'][7:]).read_bytes() + writes = [call for call in remote.calls if call[0] == 'PUT'] + assert len(writes) == 3 + assert '/manifests/' in writes[-1][1] + assert all('/blobs/uploads/' in call[1] for call in writes[:-1]) + assert remote.calls[-1][0] == 'GET' + assert all('Authorization' not in call[2] for call in remote.calls) + + +def test_registry_repeat_push_and_missing_optional_headers(layout, api): + with registry_server() as remote: + remote.digest_headers = False + client = sdk.Registry(remote.base) + first = client.publish(layout, 'team/suite') + remote.calls.clear() + assert client.publish(layout, 'team/suite') == first + assert [c[0] for c in remote.calls] == ['HEAD', 'HEAD', 'PUT', 'GET'] + + +@pytest.mark.parametrize('auth', ['basic', 'bearer']) +def test_registry_basic_and_bearer_credentials_are_origin_bound(layout, api, auth): + with registry_server() as remote, registry_server() as issuer: + remote.auth = auth + issuer.auth = 'basic' + remote.realm = issuer.base + '/token' + credentials = {remote.base: ('fixture', remote.password), issuer.base: ('fixture', issuer.password)} + reference = sdk.Registry(remote.base, credentials=credentials).publish(layout, 'team/suite') + assert reference.endswith(next(iter(remote.manifests))) + if auth == 'bearer': + assert issuer.calls and all(c[1].startswith('/token?') for c in issuer.calls) + assert all(c[2].get('Authorization', '').startswith('Basic ') for c in issuer.calls) + assert all(not c[2].get('Authorization', '').startswith('Basic ') for c in remote.calls) + else: + assert not issuer.calls + assert credentials[remote.base] == ('fixture', remote.password) + + +def test_registry_bearer_realm_does_not_inherit_registry_password(layout, api): + with registry_server() as remote, registry_server() as issuer: + remote.auth = 'bearer' + remote.realm = issuer.base + '/token' + sdk.Registry(remote.base, credentials={remote.base: ('fixture', remote.password)}).publish(layout, 'team/suite') + assert issuer.calls + assert all('Authorization' not in c[2] for c in issuer.calls) + + +def test_registry_upload_queries_and_redirects_preserve_bytes(layout, api): + with registry_server() as remote, registry_server() as storage: + remote.auth = 'basic' + remote.location = '/redirect?state=keep%2Bexact' + + def reply(method, path, headers, body): + if path.startswith('/redirect?'): + assert 'state=keep%2Bexact' in path + assert 'digest=sha256%3A' in path or 'digest=sha256:' in path + return 307, {'Location': storage.base + path.replace('/redirect?', '/v2/team/suite/blobs/uploads/fixture?')}, b'' + return None + + remote.override = reply + sdk.Registry(remote.base, credentials={remote.base: ('fixture', remote.password)}).publish(layout, 'team/suite') + assert len(storage.blobs) == 2 + assert all('Authorization' not in c[2] for c in storage.calls) + for digest, data in storage.blobs.items(): + assert digest == 'sha256:' + hashlib.sha256(data).hexdigest() + + +def test_registry_auth_replay_rewinds_body(layout, api): + with registry_server() as remote: + first = [] + + def reply(method, path, headers, body): + if method == 'PUT' and '/uploads/' in path and not first: + first.append(body) + remote.auth = 'bearer' + return 401, {'WWW-Authenticate': f'Bearer realm="{remote.base}/token",service="fixture",scope="repository:team/suite:pull,push"'}, b'' + return None + + remote.override = reply + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert first and first[0] in remote.blobs.values() + + +@pytest.mark.parametrize('invalid', ['directory', 'corrupt']) +def test_registry_refuses_before_network_on_invalid_or_changed_layout(layout, api, invalid): + if invalid == 'directory': + (layout/'oci-layout').unlink() + else: + manifest = json.loads((layout/'index.json').read_bytes())['manifests'][0] + (layout/'blobs/sha256'/manifest['digest'][7:]).write_bytes(b'changed') + with registry_server() as remote: + with pytest.raises(ValueError): + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert remote.calls == [] + + +def test_registry_snapshots_outlive_source(layout, api): + import shutil + with registry_server() as remote: + removed = [] + + def reply(method, path, headers, body): + if not removed: + shutil.rmtree(layout) + removed.append(True) + return None + + remote.override = reply + reference = sdk.Registry(remote.base).publish(layout, 'team/suite') + assert reference.endswith(next(iter(remote.manifests))) + assert len(remote.blobs) == 2 and not layout.exists() + + +@pytest.mark.parametrize('failure', ['manifest-digest', 'digest-header', 'head-size', 'final-response']) +def test_registry_uncertain_or_mismatched_remote_never_returns_reference(layout, api, failure): + with registry_server() as remote: + def reply(method, path, headers, body): + if failure == 'manifest-digest' and method == 'GET' and '/manifests/' in path: + return 200, {}, b'{}' + if failure == 'digest-header' and method == 'PUT': + return 201, {'Docker-Content-Digest': 'sha256:' + '0'*64}, b'' + if failure == 'head-size' and method == 'HEAD': + return 200, {'Content-Length': '123'}, b'' + if failure == 'final-response' and method == 'PUT' and '/manifests/' in path: + remote.manifests[path.rsplit('/', 1)[1]] = body + return 503, {}, b'no response certainty' + return None + remote.override = reply + with pytest.raises(ValueError): + sdk.Registry(remote.base).publish(layout, 'team/suite') + remote.override = lambda *args: None + assert sdk.Registry(remote.base).publish(layout, 'team/suite').endswith(next(iter(remote.manifests))) + + +def test_registry_errors_and_repr_redact_secrets(layout, api): + with registry_server() as remote: + client = sdk.Registry(remote.base, credentials={remote.base: ('fixture', remote.password)}) + remote.override = lambda *args: (500, {}, remote.password.encode()) + with pytest.raises(ValueError) as caught: + client.publish(layout, 'team/suite') + assert remote.password not in str(caught.value) + repr(client) + assert '500' in str(caught.value) + assert caught.value.__suppress_context__ + + +@pytest.mark.parametrize('repository', ['', '../suite', 'Team/suite', 'team/suite:tag', 'team//suite', 'a'*256]) +def test_registry_validates_target_and_closes_resources(layout, api, repository): + with registry_server() as remote: + with pytest.raises(ValueError): + sdk.Registry(remote.base).publish(layout, repository) + assert remote.calls == [] diff --git a/python/tests/test_registry_failures.py b/python/tests/test_registry_failures.py new file mode 100644 index 0000000..778aaef --- /dev/null +++ b/python/tests/test_registry_failures.py @@ -0,0 +1,181 @@ +import io +from contextlib import contextmanager + +import pytest +import agent_suite as sdk + +from _registry_fixture import registry_server +from test_registry import layout +from test_package import api + + +@pytest.mark.parametrize('challenge', [ + 'Bearer realm="{base}/token",realm="{base}/other"', + 'Bearer realm="{base}/token",broken', + 'Bearer service="fixture"', + 'Bearer realm="ftp://invalid.test/token"', + 'Unknown realm="fixture"', +]) +def test_registry_malformed_challenges_refuse_safely(layout, api, challenge): + with registry_server() as remote: + remote.override = lambda *args: (401, {'WWW-Authenticate': challenge.format(base=remote.base)}, b'') + with pytest.raises(ValueError): + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert len(remote.calls) == 1 + + +def test_registry_selects_bearer_from_multiple_challenges(layout, api): + with registry_server() as remote: + remote.auth = 'bearer' + + def reply(method, path, headers, body): + if not path.startswith('/token') and 'Authorization' not in headers: + challenge = f'Basic realm="fixture", Bearer realm="{remote.base}/token",service="fixture",scope="repository:team/suite:pull,push"' + return 401, {'WWW-Authenticate': challenge}, b'' + return None + + remote.override = reply + assert sdk.Registry(remote.base).publish(layout, 'team/suite') + assert len(remote.manifests) == 1 + + +def test_registry_bearer_parameters_allow_optional_whitespace(layout, api): + with registry_server() as remote: + remote.auth = 'bearer' + + def reply(method, path, headers, body): + if not path.startswith('/token') and 'Authorization' not in headers: + return 401, {'WWW-Authenticate': f'Bearer realm = "{remote.base}/token", service = "fixture", scope = "repository:team/suite:pull,push"'}, b'' + if path.startswith('/token'): + from urllib.parse import parse_qs, urlsplit + assert parse_qs(urlsplit(path).query) == { + 'service': ['fixture'], 'scope': ['repository:team/suite:pull,push']} + return None + + remote.override = reply + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert len(remote.manifests) == 1 + + +@pytest.mark.parametrize('body', [b'[]', b'{}', b'{"token":null}', b'{"token":"bad token"}', b'not-json']) +def test_registry_malformed_tokens_refuse_safely(layout, api, body): + with registry_server() as remote: + remote.auth = 'bearer' + remote.override = lambda method, path, *args: (200, {}, body) if path.startswith('/token') else None + with pytest.raises(ValueError, match='token'): + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert not remote.blobs and not remote.manifests + + +def test_registry_snapshot_rechecks_bytes_before_http(layout, api, monkeypatch): + from agent_suite import _registry + original = _registry._open_blob + + @contextmanager + def changed(root, descriptor): + with original(root, descriptor) as source: + yield io.BytesIO(b'changed' + source.read()) + + monkeypatch.setattr(_registry, '_open_blob', changed) + with registry_server() as remote: + with pytest.raises(ValueError, match='snapshot'): + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert remote.calls == [] + + +def test_registry_does_not_discover_netrc_or_cookies(layout, api, tmp_path, monkeypatch): + netrc = tmp_path/'netrc' + netrc.write_text('machine 127.0.0.1 login fixture password fake-netrc-password\n') + monkeypatch.setenv('NETRC', str(netrc)) + with registry_server() as remote: + remote.override = lambda *args: (404, {'Set-Cookie': 'fixture=cookie; Path=/'}, b'') if args[0] == 'HEAD' else None + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert all('Cookie' not in call[2] and 'Authorization' not in call[2] for call in remote.calls) + + +@pytest.mark.parametrize('fail', [False, True]) +def test_registry_releases_streams_and_session(layout, api, monkeypatch, fail): + from agent_suite import _registry + original = _registry.tempfile.TemporaryFile + streams = [] + closed = [] + original_close = _registry.HTTP.close + + def temporary(*args, **kwargs): + stream = original(*args, **kwargs) + streams.append(stream) + return stream + + def close(session): + closed.append(session) + original_close(session) + + monkeypatch.setattr(_registry.tempfile, 'TemporaryFile', temporary) + monkeypatch.setattr(_registry.HTTP, 'close', close) + with registry_server() as remote: + if fail: + remote.override = lambda *args: (500, {}, b'') + with pytest.raises(ValueError): + sdk.Registry(remote.base).publish(layout, 'team/suite') + else: + sdk.Registry(remote.base).publish(layout, 'team/suite') + assert len(closed) == 1 and len(streams) >= 2 + assert all(stream.closed for stream in streams) + + +@pytest.mark.parametrize('redirect', [False, True]) +def test_registry_idna_transport_origin_binds_credentials(layout, api, monkeypatch, redirect): + import socket + from urllib.parse import urlsplit + real_dns = socket.getaddrinfo + + def loopback_dns(host, port, *args, **kwargs): + assert host in ('127.0.0.1', 'fass.test', 'xn--fa-hia.test') + return real_dns('127.0.0.1', port, *args, **kwargs) + + monkeypatch.setattr(socket, 'getaddrinfo', loopback_dns) + with registry_server() as remote, registry_server() as issuer: + port = urlsplit(issuer.base).port + allowed = f'http://fass.test:{port}' + other = f'http://faß.test:{port}/token' + remote.auth = 'bearer' + remote.realm = allowed + '/token' if redirect else other + if redirect: + # HTTP headers carry Latin-1; Requests recovers UTF-8 Location bytes. + issuer.override = lambda method, path, headers, body: ( + 307, {'Location': other.encode('utf-8').decode('latin-1')}, b'' + ) if headers['Host'] == f'fass.test:{port}' else None + sdk.Registry(remote.base, credentials={allowed: ('fixture', issuer.password)}).publish(layout, 'team/suite') + forbidden = [c for c in issuer.calls if c[2]['Host'] == f'xn--fa-hia.test:{port}'] + assert len(forbidden) == 1 + assert all('Authorization' not in c[2] for c in forbidden) + if redirect: + assert issuer.calls[0][2]['Authorization'].startswith('Basic ') + + +@pytest.mark.parametrize('stage', ['basic', 'realm-basic', 'bearer-token']) +def test_registry_unicode_auth_failures_are_redacted(layout, api, stage): + import json + secret = 'fixture-secret-\u1200-end' + with registry_server() as remote, registry_server() as issuer: + remote.auth = 'basic' if stage == 'basic' else 'bearer' + credentials = {remote.base: ('fixture', secret)} if stage == 'basic' else {} + if stage == 'realm-basic': + remote.realm = issuer.base + '/token' + credentials[issuer.base] = ('fixture', secret) + if stage == 'bearer-token': + remote.override = lambda method, path, *args: ( + 200, {}, json.dumps({'token': secret}).encode() + ) if path.startswith('/token') else None + with pytest.raises(ValueError) as caught: + sdk.Registry(remote.base, credentials=credentials).publish(layout, 'team/suite') + assert secret not in str(caught.value) + repr(caught.value) + assert not isinstance(caught.value, UnicodeError) + assert caught.value.__suppress_context__ + assert not remote.blobs and not remote.manifests + + +@pytest.mark.parametrize('address', ['https://fixture:password@registry.test', 'https://registry.test/path', 'https://registry.test?secret=query', 'file:///path', '']) +def test_registry_invalid_origins_refuse(address): + with pytest.raises(ValueError): + sdk.Registry(address) diff --git a/python/tests/test_release.py b/python/tests/test_release.py new file mode 100644 index 0000000..80fa334 --- /dev/null +++ b/python/tests/test_release.py @@ -0,0 +1,240 @@ +import hashlib +import importlib.util +import io +import json +import tarfile +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import pytest +from test_documents import ROOT + + +@pytest.fixture +def release(): + path = ROOT/'scripts/check-python-release.py' + assert path.is_file(), 'missing Python release qualification helper' + spec = importlib.util.spec_from_file_location('python_release', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def candidate(tmp_path, monkeypatch, release): + sdk, contract, catalog = (tmp_path/name for name in ('sdk', 'contract', 'catalog')) + sdk_sha, contract_sha, catalog_sha = '1'*40, '2'*40, '3'*40 + schemas = {p.name: p.read_bytes().replace(b'/schema/draft/', b'/schema/v0.8/') for p in (ROOT/'schema').glob('*.schema.json')} + inventory = {name: {'id': json.loads(data)['$id'], 'sha256': hashlib.sha256(data).hexdigest()} for name, data in schemas.items()} + provenance = {'schemaRevision': sdk_sha, 'schemas': {name: {'id': row['id'], 'digest': 'sha256:'+row['sha256']} for name, row in inventory.items()}} + for repo in (sdk, contract, catalog): + (repo/'schema').mkdir(parents=True) + for name, data in schemas.items(): + (repo/'schema'/name).write_bytes(data) + entry = {'path': 'v0.8', 'release': 'v0.8.0', 'sourceCommit': contract_sha, + 'assetUrl': 'https://github.com/agent-primitives/suite/releases/download/v0.8.0/schemas-v0.8.0.json', + 'sha256': '4'*64, 'schemas': inventory} + registry = {'version': 1, 'bundles': [entry], 'releases': {'v0.8.0': 'v0.8'}} + # Catalog is intentionally later than both frozen tags; its current schema tree is draft. + for name in schemas: + (catalog/'schema'/name).write_bytes((ROOT/'schema'/name).read_bytes()) + (catalog/'docs').mkdir() + (catalog/'docs/schema-catalog.json').write_text(json.dumps(registry)) + publication = tmp_path/'dist' + publication.mkdir() + wheel = publication/'agent_suite-0.1.0-py3-none-any.whl' + derived = tmp_path/'derived.whl' + sdist = publication/'agent_suite-0.1.0.tar.gz' + + def distributions(version='0.1.0', derived_provenance=None): + metadata = f'Name: agent-suite\nVersion: {version}\n' + for target in (wheel, derived): + with zipfile.ZipFile(target, 'w') as archive: + archive.writestr(f'agent_suite-{version}.dist-info/METADATA', metadata) + for name, data in schemas.items(): + archive.writestr('agent_suite/_schemas/'+name, data) + archive.writestr('agent_suite/_schemas/provenance.json', json.dumps(derived_provenance if target == derived and derived_provenance is not None else provenance)) + with tarfile.open(sdist, 'w:gz') as archive: + entries = {'PKG-INFO': metadata.encode(), '_schema_source/provenance.json': json.dumps(provenance).encode()} + entries.update({'_schema_source/'+name: data for name, data in schemas.items()}) + for name, data in entries.items(): + info = tarfile.TarInfo(f'agent_suite-{version}/'+name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + + distributions() + tags = {'refs/tags/python/v0.1.0^{commit}': sdk_sha, 'refs/tags/v0.8.0^{commit}': contract_sha} + + def git(repo, *args): + repo = Path(repo) + if args == ('rev-parse', 'HEAD'): + return {sdk: sdk_sha, contract: contract_sha, catalog: catalog_sha}[repo].encode() + if args[0] == 'rev-parse': + return tags[args[-1]].encode() + if args[0] == 'show': + ref, name = args[1].split(':', 1) + if name == 'docs/schema-catalog.json': + return (catalog/name).read_bytes() + return (repo/name).read_bytes() + raise AssertionError(args) + + monkeypatch.setattr(release, 'git', git) + monkeypatch.setenv('AGENTSUITE_GO_ORACLE_ROOT', str(contract)) + options = SimpleNamespace(sdk_repo=sdk, contract_repo=contract, catalog_repo=catalog, + catalog_ref=catalog_sha, sdk_tag='python/v0.1.0', contract_tag='v0.8.0', + wheel=wheel, sdist=sdist, sdist_wheel=derived, out=tmp_path/'qualification.json') + return SimpleNamespace(options=options, schemas=schemas, provenance=provenance, + inventory=inventory, distributions=distributions, tags=tags, + dist=publication, sdk=sdk, contract=contract, catalog=catalog) + + +@pytest.mark.parametrize('bad', ['development', 'development-tag', 'sdk-tag', 'contract-tag', 'retargeted-sdk', 'retargeted-contract']) +def test_python_release_rejects_development_or_wrong_tags(candidate, release, bad): + options = candidate.options + if bad == 'development': + candidate.distributions(version='0.1.0.dev0') + elif bad == 'development-tag': + options.sdk_tag = 'python/v0.1.0.dev0' + elif bad == 'sdk-tag': + options.sdk_tag = 'python/main' + elif bad == 'contract-tag': + options.contract_tag = 'v0.7.0' + elif bad == 'retargeted-sdk': + candidate.tags['refs/tags/python/v0.1.0^{commit}'] = '9'*40 + else: + candidate.tags['refs/tags/v0.8.0^{commit}'] = '9'*40 + with pytest.raises(ValueError): + release.qualify(options) + + +@pytest.mark.parametrize('bad', ['schema', 'draft', 'catalog', 'oracle']) +def test_python_release_binds_exact_frozen_contract_and_schema_bytes(candidate, release, bad, monkeypatch): + if bad == 'schema': + path = candidate.sdk/'schema/suite.schema.json' + path.write_bytes(path.read_bytes()+b'\n') + elif bad == 'draft': + for repo in (candidate.sdk, candidate.contract): + path = repo/'schema/suite.schema.json' + path.write_bytes(path.read_bytes().replace(b'/v0.8/', b'/draft/')) + elif bad == 'catalog': + (candidate.catalog/'docs/schema-catalog.json').write_text('{"version":1,"bundles":[],"releases":{}}') + else: + monkeypatch.setenv('AGENTSUITE_GO_ORACLE_ROOT', str(candidate.sdk)) + with pytest.raises(ValueError): + release.qualify(candidate.options) + + +def test_python_release_checks_both_distribution_provenances(candidate, release): + changed = dict(candidate.provenance, schemaRevision='9'*40) + candidate.distributions(derived_provenance=changed) + with pytest.raises(ValueError, match='provenance'): + release.qualify(candidate.options) + + +def test_python_release_records_validator_and_artifact_identities(candidate, release): + record = release.qualify(candidate.options) + assert record['sdkRevision'] == '1'*40 + assert record['suiteRevision'] == record['validatorRevision'] == record['vectorsRevision'] == '2'*40 + assert record['catalogRevision'] == '3'*40 + assert record['schemas'] == candidate.inventory + assert len(record['distributions']) == 2 + candidate.options.out.write_text(json.dumps(record)) + release.verify_artifacts(candidate.options.out, candidate.dist) + candidate.options.wheel.write_bytes(b'changed') + with pytest.raises(ValueError): + release.verify_artifacts(candidate.options.out, candidate.dist) + + +@pytest.mark.parametrize('change', ['missing', 'extra', 'tampered']) +def test_python_release_rejects_downloaded_artifact_changes(candidate, release, change): + candidate.options.out.write_text(json.dumps(release.qualify(candidate.options))) + if change == 'missing': + candidate.options.wheel.unlink() + elif change == 'extra': + (candidate.dist/'unqualified.whl').write_bytes(b'extra') + else: + candidate.options.sdist.write_bytes(b'changed') + with pytest.raises(ValueError): + release.verify_artifacts(candidate.options.out, candidate.dist) + + +def test_python_release_oracle_override_selects_build_and_run(tmp_path, tmp_path_factory, monkeypatch): + import test_write_conformance as oracle + selected = tmp_path/'frozen-contract' + calls = [] + + def run(*args, **kwargs): + calls.append(kwargs['cwd']) + return SimpleNamespace(returncode=0, stdout='--- PASS: TestPythonWriterOutput (0.01s)', stderr='') + + monkeypatch.setattr(oracle, 'ORACLE_ROOT', selected, raising=False) + monkeypatch.setattr(oracle.subprocess, 'run', run) + executable = oracle.writer_oracle.__wrapped__(tmp_path_factory) + oracle.go_validate(executable, tmp_path, [{'valid': True}]) + assert calls == [selected, selected/'internal/artifact'] + + +@pytest.mark.parametrize('module_name, function_name, needs_path, compiled', [ + ('test_codec', 'test_go_numeric_differential', False, False), + ('test_directory', 'test_go_casefold_differential', False, False), + ('test_archive_differential', 'test_go_archive_differential', True, True), + ('test_oci_differential', 'test_go_oci_differential', True, True), + ('test_regex', 'test_go_regex_differential', True, True), +]) +def test_python_release_all_differentials_use_selected_contract(tmp_path, monkeypatch, module_name, function_name, needs_path, compiled): + import importlib + import subprocess + module = importlib.import_module(module_name) + selected = tmp_path/'frozen-contract' + calls = [] + + class Captured(Exception): + pass + + def run(command, **kwargs): + calls.append((command, kwargs['cwd'])) + if compiled and len(calls) == 1: + return SimpleNamespace(returncode=0) + raise Captured + + monkeypatch.setattr(module, 'ORACLE_ROOT', selected, raising=False) + monkeypatch.setattr(subprocess, 'run', run) + with pytest.raises(Captured): + getattr(module, function_name)(tmp_path) if needs_path else getattr(module, function_name)() + assert calls[0][1] == selected + if compiled: + assert calls[1][1] == selected or calls[1][1] == selected/'internal/artifact' + else: + source = Path(calls[0][0][2]) + assert (source if source.is_absolute() else selected/source).is_relative_to(selected) + + +def test_python_release_qualifies_frozen_vector_suite(): + workflow = (ROOT/'.github/workflows/python-release.yml').read_text() + assert 'python -m pytest contract-source/python/tests -q' in workflow + + +def test_python_release_unmapped_maintenance_tag_refused(candidate, release): + candidate.options.contract_tag = 'v0.8.1' + candidate.tags['refs/tags/v0.8.1^{commit}'] = '2'*40 + with pytest.raises(ValueError, match='catalog'): + release.qualify(candidate.options) + + +def test_python_release_accepts_frozen_maintenance_tags(candidate, release): + # No main-ancestry restriction: exact canonical tags are the maintained authority. + candidate.options.sdk_tag = 'python/v0.1.1' + candidate.options.contract_tag = 'v0.8.1' + candidate.tags.update({'refs/tags/python/v0.1.1^{commit}': '1'*40, 'refs/tags/v0.8.1^{commit}': '2'*40}) + catalog_path = candidate.catalog/'docs/schema-catalog.json' + catalog = json.loads(catalog_path.read_bytes()) + catalog['releases']['v0.8.1'] = 'v0.8' + catalog_path.write_text(json.dumps(catalog)) + candidate.distributions(version='0.1.1') + for key in ('wheel', 'sdist'): + previous = getattr(candidate.options, key) + target = previous.with_name(previous.name.replace('0.1.0', '0.1.1')) + previous.rename(target) + setattr(candidate.options, key, target) + assert release.qualify(candidate.options)['version'] == '0.1.1' diff --git a/python/tests/test_write_conformance.py b/python/tests/test_write_conformance.py index a87d080..6e60291 100644 --- a/python/tests/test_write_conformance.py +++ b/python/tests/test_write_conformance.py @@ -14,6 +14,7 @@ CASES = json.loads((ROOT/'testdata/conformance/write-cases.json').read_bytes())['cases'] +ORACLE_ROOT = Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', str(ROOT))).resolve() @pytest.fixture(scope='session') @@ -21,7 +22,7 @@ def writer_oracle(tmp_path_factory): root = tmp_path_factory.mktemp('writer-oracle') executable = root/('oracle.exe' if os.name == 'nt' else 'oracle') subprocess.run(['go', 'test', '-c', '-o', str(executable), './internal/artifact'], - cwd=ROOT, check=True, capture_output=True, timeout=120) + cwd=ORACLE_ROOT, check=True, capture_output=True, timeout=120) return executable @@ -29,7 +30,7 @@ def go_validate(executable, root, rows): request = root/'write-oracle.json' request.write_text(json.dumps(rows), encoding='utf-8') completed = subprocess.run([str(executable), '-test.run=TestPythonWriterOutput', '-test.count=1', '-test.v'], - cwd=ROOT/'internal/artifact', env={**os.environ, 'AGENTSUITE_WRITE_ORACLE': str(request)}, + cwd=ORACLE_ROOT/'internal/artifact', env={**os.environ, 'AGENTSUITE_WRITE_ORACLE': str(request)}, text=True, capture_output=True, timeout=120) assert completed.returncode == 0, completed.stdout + completed.stderr assert '--- PASS: TestPythonWriterOutput ' in completed.stdout, completed.stdout diff --git a/scripts/check-python-release.py b/scripts/check-python-release.py new file mode 100644 index 0000000..8db4999 --- /dev/null +++ b/scripts/check-python-release.py @@ -0,0 +1,176 @@ +"""Owner-run release qualification; no network, tag creation or upload.""" +import argparse +import email.parser +import hashlib +import json +import os +import re +import subprocess +import tarfile +import zipfile +from pathlib import Path + + +SHA = re.compile(r'[0-9a-f]{40}') +SDK_TAG = re.compile(r'python/v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)') +CONTRACT_TAG = re.compile(r'v0\.8\.(0|[1-9]\d*)') + + +def git(repo, *args): + return subprocess.check_output(['git', '-C', str(repo), *args], stderr=subprocess.PIPE) + + +def revision(repo, ref): + value = git(repo, 'rev-parse', '--verify', '--end-of-options', ref+'^{commit}').decode().strip() + if not SHA.fullmatch(value): + raise ValueError('invalid release source revision') + return value + + +def inventory(repo, commit): + files = {} + for path in sorted((repo/'schema').glob('*.schema.json')): + data = path.read_bytes() + if data != git(repo, 'show', commit+':schema/'+path.name): + raise ValueError('schema checkout differs from tagged bytes') + document = json.loads(data) + identity = document['$id'] + if identity != 'https://suite.agent-primitives.dev/schema/v0.8/'+path.name: + raise ValueError('release requires frozen v0.8 schema IDs') + files[path.name] = data + if not files: + raise ValueError('release schema inventory is empty') + return files + + +def distribution(path, wheel): + if wheel: + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise ValueError('duplicate distribution member') + metadata = [n for n in names if n.endswith('.dist-info/METADATA')] + if len(metadata) != 1: + raise ValueError('invalid wheel metadata') + info = archive.read(metadata[0]) + files = {n.removeprefix('agent_suite/_schemas/'): archive.read(n) + for n in names if n.startswith('agent_suite/_schemas/') and not n.endswith('/')} + else: + with tarfile.open(path, 'r:gz') as archive: + members = archive.getmembers() + names = [m.name for m in members] + if len(names) != len(set(names)): + raise ValueError('duplicate distribution member') + metadata = [m for m in members if m.name.count('/') == 1 and m.name.endswith('/PKG-INFO')] + if len(metadata) != 1: + raise ValueError('invalid sdist metadata') + stream = archive.extractfile(metadata[0]) + if stream is None: + raise ValueError('missing sdist metadata') + info = stream.read() + files = {} + for member in members: + if '/_schema_source/' in member.name and member.isfile(): + stream = archive.extractfile(member) + files[member.name.split('/_schema_source/', 1)[1]] = stream.read() + metadata = email.parser.BytesParser().parsebytes(info) + if metadata['Name'] != 'agent-suite': + raise ValueError('distribution name mismatch') + return metadata['Version'], files + + +def qualify(options): + if not SDK_TAG.fullmatch(options.sdk_tag) or not CONTRACT_TAG.fullmatch(options.contract_tag): + raise ValueError('release requires exact SDK and frozen v0.8 root tags') + if not SHA.fullmatch(options.catalog_ref): + raise ValueError('catalog requires a pinned main revision') + sdk = Path(options.sdk_repo).resolve() + contract = Path(options.contract_repo).resolve() + catalog = Path(options.catalog_repo).resolve() + sdk_sha = git(sdk, 'rev-parse', 'HEAD').decode().strip() + suite_sha = git(contract, 'rev-parse', 'HEAD').decode().strip() + if (sdk_sha != revision(catalog, 'refs/tags/'+options.sdk_tag) + or suite_sha != revision(catalog, 'refs/tags/'+options.contract_tag) + or git(catalog, 'rev-parse', 'HEAD').decode().strip() != options.catalog_ref): + raise ValueError('release checkout/tag/catalog revision mismatch') + schemas = inventory(sdk, sdk_sha) + if schemas != inventory(contract, suite_sha): + raise ValueError('SDK schema bytes differ from selected contract') + rows = {name: {'id': json.loads(data)['$id'], 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in schemas.items()} + registry = json.loads(git(catalog, 'show', options.catalog_ref+':docs/schema-catalog.json')) + entries = [entry for entry in registry['bundles'] if entry['path'] == 'v0.8'] + if len(entries) != 1 or entries[0]['schemas'] != rows: + raise ValueError('reviewed catalog/schema inventory mismatch') + entry = entries[0] + if (entry['release'] != 'v0.8.0' or entry['sourceCommit'] != revision(catalog, 'refs/tags/v0.8.0') + or registry['releases'].get('v0.8.0') != 'v0.8' + or registry['releases'].get(options.contract_tag) != 'v0.8'): + raise ValueError('catalog/root tag binding mismatch') + if Path(os.environ.get('AGENTSUITE_GO_ORACLE_ROOT', '')).resolve() != contract: + raise ValueError('frozen contract Go oracle root mismatch') + version = options.sdk_tag[len('python/v'):] + provenance = {'schemaRevision': sdk_sha, 'schemas': {name: {'id': row['id'], 'digest': 'sha256:'+row['sha256']} for name, row in rows.items()}} + expected = dict(schemas, **{'provenance.json': None}) + for path, is_wheel in ((options.wheel, True), (options.sdist, False), (options.sdist_wheel, True)): + found_version, files = distribution(path, is_wheel) + if found_version != version: + raise ValueError('SDK tag/version mismatch or development version') + if set(files) != set(expected) or any(files[name] != data for name, data in schemas.items()): + raise ValueError('distribution schema byte inventory mismatch') + if json.loads(files['provenance.json']) != provenance: + raise ValueError('distribution schema provenance mismatch') + if (not options.wheel.name.startswith('agent_suite-'+version+'-') + or options.sdist.name != 'agent_suite-'+version+'.tar.gz'): + raise ValueError('publication filename/version mismatch') + distributions = {Path(path).name: file_digest(path) for path in (options.wheel, options.sdist)} + return {'version': version, 'sdkTag': options.sdk_tag, 'sdkRevision': sdk_sha, + 'suiteTag': options.contract_tag, 'suiteRevision': suite_sha, + 'catalogRevision': options.catalog_ref, 'schemas': rows, + 'schemaAsset': {'url': entry['assetUrl'], 'sha256': entry['sha256']}, + 'validatorRevision': suite_sha, 'vectorsRevision': suite_sha, + 'sdistWheelSha256': file_digest(options.sdist_wheel), 'distributions': distributions} + + +def file_digest(path): + digest = hashlib.sha256() + with open(path, 'rb') as stream: + while data := stream.read(128 << 10): + digest.update(data) + return digest.hexdigest() + + +def verify_artifacts(qualification, dist): + record = json.loads(Path(qualification).read_bytes()) + expected = record['distributions'] + actual = {p.name for p in Path(dist).iterdir()} + if actual != set(expected): + raise ValueError('publication distribution inventory mismatch') + for name, digest in expected.items(): + if Path(name).name != name or file_digest(Path(dist)/name) != digest: + raise ValueError('publication artifact hash mismatch') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + check = commands.add_parser('qualify') + for name in ('sdk-repo', 'contract-repo', 'catalog-repo', 'wheel', 'sdist', 'sdist-wheel', 'out'): + check.add_argument('--'+name, type=Path, required=True) + for name in ('sdk-tag', 'contract-tag', 'catalog-ref'): + check.add_argument('--'+name, required=True) + verify = commands.add_parser('verify-artifacts') + verify.add_argument('--qualification', type=Path, required=True) + verify.add_argument('--dist', type=Path, required=True) + options = parser.parse_args() + try: + if options.command == 'qualify': + options.out.write_text(json.dumps(qualify(options), sort_keys=True, indent=2)+'\n') + else: + verify_artifacts(options.qualification, options.dist) + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error: + parser.exit(1, 'python-release: '+str(error)+'\n') + + +if __name__ == '__main__': + main()