diff --git a/.gitignore b/.gitignore index b52e204e72..dca4334122 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ __pycache__/ .venv/ .ruff_cache/ .pytest_cache/ +cache/ diff --git a/python/lean_pool/aggregator/__init__.py b/python/lean_pool/aggregator/__init__.py new file mode 100644 index 0000000000..a1a84e6072 --- /dev/null +++ b/python/lean_pool/aggregator/__init__.py @@ -0,0 +1 @@ +"""Aggregation tooling that fetches Reservoir data and filters Lean packages.""" diff --git a/python/lean_pool/aggregator/__main__.py b/python/lean_pool/aggregator/__main__.py new file mode 100644 index 0000000000..de236a4299 --- /dev/null +++ b/python/lean_pool/aggregator/__main__.py @@ -0,0 +1,39 @@ +"""CLI entry point for the aggregator. + +Run with ``uv run python -m lean_pool.aggregator`` from ``python/``. +""" + +from pathlib import Path + +import click + +from lean_pool.aggregator.reservoir import ( + MANIFEST_URL, + fetch_manifest, + save_manifest, +) + + +@click.command() +@click.option( + "--url", + default=MANIFEST_URL, + show_default=True, + help="Reservoir manifest URL.", +) +@click.option( + "--output", + type=click.Path(dir_okay=False, path_type=Path), + default=Path("cache/manifest.json"), + show_default=True, + help="Where to save the downloaded manifest.", +) +def main(url: str, output: Path) -> None: + """Download the Reservoir manifest and save it to disk.""" + manifest = fetch_manifest(url) + save_manifest(manifest, output) + click.echo(f"Saved {len(manifest['packages'])} packages to {output}") + + +if __name__ == "__main__": + main() diff --git a/python/lean_pool/aggregator/reservoir.py b/python/lean_pool/aggregator/reservoir.py new file mode 100644 index 0000000000..05ebadbea8 --- /dev/null +++ b/python/lean_pool/aggregator/reservoir.py @@ -0,0 +1,165 @@ +"""Fetch and decode the Reservoir package manifest. + +The Reservoir website bundles every indexed Lean package into a single +JSON file at ``https://reservoir.lean-lang.org/index/manifest.json``. +That file is the canonical input for aggregation: it contains every +package's metadata, recent versions, and per-toolchain build results. + +The TypedDicts below mirror Reservoir's own type definitions in +``reservoir/scripts/utils/`` (``package.py``, ``manifest.py``, +``toolchain.py``) so the decoded manifest is statically typed end to +end, not just ``dict[str, Any]``. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TypedDict +from urllib.request import urlopen + +logger = logging.getLogger(__name__) + +MANIFEST_URL = "https://reservoir.lean-lang.org/index/manifest.json" +DEFAULT_TIMEOUT_SECONDS = 60 + + +class PackageSource(TypedDict, total=False): + """A source location for a package. Most entries are GitHub repos.""" + + type: str + host: str + id: str + fullName: str + repoUrl: str + gitUrl: str + defaultBranch: str + + +class BuildResult(TypedDict): + """Outcome of building a package against a single toolchain.""" + + built: bool | None + tested: bool | None + toolchain: str + requiredUpdate: bool | None + archiveSize: int | None + archiveHash: str | None + runAt: str + url: str | None + + +class Build(BuildResult): + """A build result tagged with the source revision it ran against.""" + + revision: str + + +class DependencyBase(TypedDict): + """Fields shared by outgoing dependencies and incoming dependents.""" + + type: str + name: str + scope: str | None + version: str + transitive: bool | None + rev: str | None + inputRev: str | None + url: str | None + + +class Dependency(DependencyBase, total=False): + """An outgoing dependency. ``fullName`` is set when resolvable.""" + + fullName: str + + +class Dependent(DependencyBase): + """A package that depends on this one. ``fullName`` is always set.""" + + fullName: str + + +class PackageVersion(TypedDict): + """A released version of a package, as serialized in the manifest.""" + + version: str + revision: str + date: str + tag: str | None + toolchain: str | None + platformIndependent: bool | None + license: str | None + licenseFiles: list[str] + readmeFile: str | None + dependencies: list[Dependency] + + +class Package(TypedDict): + """A package as published in the Reservoir manifest.""" + + name: str + owner: str + fullName: str + description: str | None + keywords: list[str] | None + homepage: str | None + license: str | None + createdAt: str + updatedAt: str + stars: int + sources: list[PackageSource] + versions: list[PackageVersion] + dependents: list[Dependent] + builds: list[Build] + + +class Toolchain(TypedDict): + """An indexed Lean toolchain release.""" + + name: str + version: int | None + tag: str + date: str + releaseUrl: str + prerelease: bool + + +class ReservoirManifest(TypedDict): + """The decoded ``manifest.json`` produced by Reservoir's bundler.""" + + bundledAt: str + toolchains: list[Toolchain] + packages: list[Package] + packageAliases: dict[str, str] + + +def fetch_manifest( + url: str = MANIFEST_URL, timeout: float = DEFAULT_TIMEOUT_SECONDS +) -> ReservoirManifest: + """Download and parse the Reservoir manifest JSON. + + Args: + url: The URL to fetch the manifest from. + timeout: Socket timeout in seconds for the HTTP request. + + Returns: + The parsed manifest with ``bundledAt``, ``toolchains``, + ``packages``, and ``packageAliases`` fields. + """ + logger.info("Fetching %s", url) + with urlopen(url, timeout=timeout) as response: + return json.load(response) + + +def save_manifest(manifest: ReservoirManifest, path: Path) -> None: + """Write the manifest to disk as pretty-printed JSON. + + Args: + manifest: The parsed manifest. + path: The output file path; parent directories are created. + """ + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as output_file: + json.dump(manifest, output_file, indent=2) diff --git a/python/pyproject.toml b/python/pyproject.toml index 2789a3c227..14b2db8788 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,9 @@ version = "0.0.1" description = "Python tooling for the lean-pool repository (Reservoir aggregation, candidate filtering)." readme = "README.md" requires-python = ">=3.13" -dependencies = [] +dependencies = [ + "click>=8.1", +] [dependency-groups] diff --git a/python/tests/aggregator/__init__.py b/python/tests/aggregator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/tests/aggregator/test_reservoir.py b/python/tests/aggregator/test_reservoir.py new file mode 100644 index 0000000000..f8f1ebcba8 --- /dev/null +++ b/python/tests/aggregator/test_reservoir.py @@ -0,0 +1,41 @@ +"""Tests for the Reservoir manifest fetcher and writer.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from unittest.mock import patch + +from lean_pool.aggregator.reservoir import ( + MANIFEST_URL, + fetch_manifest, + save_manifest, +) + + +def test_save_manifest_round_trip(tmp_path: Path) -> None: + """save_manifest writes JSON that can be read back identically.""" + manifest = {"bundledAt": "2026-05-09T00:00:00Z", "packages": [{"stars": 3}]} + output_path = tmp_path / "nested" / "manifest.json" + + save_manifest(manifest, output_path) + + assert output_path.exists() + assert json.loads(output_path.read_text()) == manifest + + +def test_fetch_manifest_parses_response() -> None: + """fetch_manifest decodes JSON returned by urlopen.""" + payload = {"bundledAt": "now", "packages": []} + response = io.BytesIO(json.dumps(payload).encode()) + response.__enter__ = lambda self: self # type: ignore[method-assign] + response.__exit__ = lambda self, *exc: None # type: ignore[method-assign] + + with patch( + "lean_pool.aggregator.reservoir.urlopen", return_value=response + ) as mock_urlopen: + result = fetch_manifest(timeout=5) + + assert result == payload + mock_urlopen.assert_called_once_with(MANIFEST_URL, timeout=5) diff --git a/python/uv.lock b/python/uv.lock index 9dba789105..78f8b7c559 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]] @@ -32,6 +32,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, ] +[[package]] +name = "click" +version = "8.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -232,6 +244,9 @@ wheels = [ name = "lean-pool-tools" version = "0.0.1" source = { virtual = "." } +dependencies = [ + { name = "click" }, +] [package.dev-dependencies] dev = [ @@ -246,6 +261,7 @@ test = [ ] [package.metadata] +requires-dist = [{ name = "click", specifier = ">=8.1" }] [package.metadata.requires-dev] dev = [{ name = "ruff", specifier = ">=0.9" }]