From 3325d053e31e2caf4aa7c379c2d0151e9af86c36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 19:02:43 +0000 Subject: [PATCH 1/5] feat: support use as a Python library nf-docs stays a CLI tool first, but the core was already side-effect-clean (all rich output and sys.exit calls live in cli.py), so exposing a supported library API was mostly a matter of packaging and documenting what exists. Adds a small facade over the existing classes: import nf_docs pipeline = nf_docs.extract("./my_pipeline") markdown = nf_docs.render(pipeline, "markdown") files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") extract() and render() mirror PipelineExtractor/get_renderer; generate() always writes files and returns their paths. The CLI's habit of streaming json/yaml to stdout stays in cli.py, since that's a command-line concern. Output path policy (format aliases, single-module auto-detection, default filenames) moves out of cli.py into nf_docs.output so both entry points share one implementation. cli.py becomes a presentation layer over it and is otherwise unchanged - tests/test_cli.py passes untouched, and CLI and library output was verified byte-identical across all five formats. PipelineExtractor now takes an optional config= argument. Library callers get NfDocsConfig() defaults rather than the user's ~/.config/nf-docs/config.yaml, so programmatic use is reproducible; the CLI passes load_config() explicitly to keep its behaviour. The global get_config()/reset_config() remain. Also adds a py.typed marker, an explicit __all__ covering the facade, models, renderers, progress types and exceptions, and a "Python API" docs page. Note: most NfDocsConfig fields (include_hidden_params, max_readme_length, strip_readme_badges, exclude_patterns, ignore_input_prefixes, default_format) are not actually consulted during extraction - the behaviours exist but are hardcoded. Only ignore_config_prefixes takes effect. The docs page documents just that one so the library doesn't advertise inert options; wiring up the rest is left for a separate change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qwciLzAxtZMwu3uWnA24H --- AGENTS.md | 11 +- CHANGELOG.md | 19 +++ MANIFEST.in | 1 + README.md | 17 +++ docs/python-api.md | 320 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- src/nf_docs/__init__.py | 58 ++++++- src/nf_docs/api.py | 221 +++++++++++++++++++++++++++ src/nf_docs/cli.py | 127 ++++------------ src/nf_docs/extractor.py | 9 +- src/nf_docs/output.py | 145 ++++++++++++++++++ src/nf_docs/py.typed | 0 tests/test_api.py | 251 ++++++++++++++++++++++++++++++ zensical.toml | 1 + 14 files changed, 1077 insertions(+), 105 deletions(-) create mode 100644 docs/python-api.md create mode 100644 src/nf_docs/api.py create mode 100644 src/nf_docs/output.py create mode 100644 src/nf_docs/py.typed create mode 100644 tests/test_api.py diff --git a/AGENTS.md b/AGENTS.md index 1728e58..8b4b48d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -178,9 +178,11 @@ Async test mode is `auto` (via `asyncio_mode = "auto"` in pyproject.toml). ``` src/nf_docs/ -├── __init__.py # Version from importlib.metadata, re-exports models +├── __init__.py # Version, public API re-exports (__all__) +├── api.py # High-level library API: extract() / render() / generate() ├── cache.py # XDG-compliant caching with content hashing ├── cli.py # CLI entry point (rich-click) +├── output.py # Output path policy shared by the CLI and api.py ├── config.py # User config from ~/.config/nf-docs/config.yaml ├── config_parser.py # Parse nextflow.config via `nextflow config -flat` ├── extractor.py # Main extraction orchestration @@ -219,6 +221,13 @@ src/nf_docs/ new output format means updating `get_renderer()`, the CLI's `--format` choices, and tests. - **Config parsing**: `config_parser.py` shells out to `nextflow config -flat`. `nextflow_env.py` sets an isolated `NXF_HOME` so this doesn't pollute the user's Nextflow state. +- **Two entry points**: `cli.py` (rich-click) and `api.py` (library). Both go through `output.py` + for path/format decisions, so behaviour stays in sync. Keep console output and `sys.exit()` in + `cli.py` only — core modules log via `logging` and raise exceptions. The CLI reads the user's + `~/.config/nf-docs/config.yaml` and passes it in explicitly; `api.py` uses `NfDocsConfig()` + defaults so library calls are reproducible. +- **Public API**: everything in `nf_docs.__all__` plus `nf_docs.models`. Adding to it is a + commitment — update `docs/python-api.md` and `tests/test_api.py` alongside. - **Pre-commit hook entry**: `.pre-commit-hooks.yaml` exposes the CLI as the `nf-docs` hook with default `args: [., --format, html]`, triggered by changes to `.nf`, `nextflow.config`, `nextflow_schema.json`, `meta.yml`, or README files. Guarded by `tests/test_pre_commit_hooks.py`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fe9c27..dd1b6bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## Unreleased + +### Added + +- Supported Python API for using nf-docs as a library: `nf_docs.extract()`, `nf_docs.render()` and + `nf_docs.generate()`, alongside a wider set of exports from the top-level `nf_docs` namespace +- `py.typed` marker so type hints reach downstream users +- `PipelineExtractor` accepts a `config=` argument, so callers can supply an `NfDocsConfig` instead + of relying on the global instance +- New "Python API" documentation page + +### Changed + +- Output path policy (format aliases, single-module detection, default filenames) moved from + `cli.py` into a new `nf_docs.output` module so the CLI and the Python API share one implementation +- Library callers now get `NfDocsConfig()` defaults rather than the user's + `~/.config/nf-docs/config.yaml`, which the CLI continues to read. This keeps programmatic use + reproducible. + ## [Version 0.4.0](https://github.com/ewels/nf-docs/releases/tag/v0.4.0) - 2026-05-29 ### Added diff --git a/MANIFEST.in b/MANIFEST.in index ff8bc3d..07edd72 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ recursive-include src/nf_docs/templates *.html *.css +include src/nf_docs/py.typed diff --git a/README.md b/README.md index 008a108..1b21312 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,23 @@ nf-docs generate modules/mytool/subtools/main.nf --format md By default this writes `README.md` next to `main.nf`. Use `--output` to pick a different path, or `--format json` / `--format yaml` to print structured output to stdout instead. +## Use as a Python library + +`nf-docs` is a CLI tool first, but it's also an importable Python package — useful for feeding +pipeline data into your own tooling or generating docs as part of a site build: + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") +print([p.name for p in pipeline.processes]) + +markdown = nf_docs.render(pipeline, "markdown") +files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") +``` + +See the [Python API docs](https://ewels.github.io/nf-docs/python-api/) for the full reference. + ## Prek / pre-commit hook `nf-docs` can generate pipeline documentation automatically before each commit using diff --git a/docs/python-api.md b/docs/python-api.md new file mode 100644 index 0000000..adbf8f0 --- /dev/null +++ b/docs/python-api.md @@ -0,0 +1,320 @@ +# Python API + +`nf-docs` is a CLI tool first, but it's a normal Python package too. If you need something the CLI +doesn't cover — feeding pipeline data into your own tooling, rendering docs as part of a site build, +or documenting modules in a loop — you can import it directly. + +```bash +pip install nf-docs +``` + +The same requirements apply as for the CLI: Java for the Nextflow Language Server, and optionally +Nextflow itself for parsing `nextflow.config`. + +## Three functions + +Most things are covered by three functions: + +| Function | Does | Returns | +| ------------ | ------------------------------------------------------- | ------------ | +| `extract()` | Reads a pipeline (or a single module) into a data model | `Pipeline` | +| `render()` | Turns a `Pipeline` into a string in a given format | `str` | +| `generate()` | Both of the above, then writes the result to disk | `list[Path]` | + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") +markdown = nf_docs.render(pipeline, "markdown") +files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") +``` + +Unlike the CLI, these never print to the console and never exit the process — they return values and +raise exceptions. + +## Reading pipeline data + +`extract()` gives you a [`Pipeline`](#the-pipeline-model) object. This is the one to reach for when +you want the data rather than the documentation. + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") + +print(pipeline.metadata.name, pipeline.metadata.version) + +for process in pipeline.processes: + print(f"{process.name}: {process.docstring}") + +# Required parameters that have no default +for param in pipeline.inputs: + if param.required and param.default is None: + print(f"{param.name} ({param.type}) - {param.description}") +``` + +Every model has a `to_dict()`, so handing the whole thing to something else is one call: + +```python +import json + +with open("pipeline-api.json", "w") as fh: + json.dump(pipeline.to_dict(), fh, indent=2) +``` + +## Rendering into an existing build + +`render()` returns a string, which is usually what you want when another tool owns the output +directory — a static site generator, a template, a wiki upload. + +```python +from pathlib import Path +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") + +Path("src/content/docs/api.md").write_text( + "---\ntitle: Pipeline API\n---\n\n" + nf_docs.render(pipeline, "markdown") +) +``` + +Renderer-specific options are passed straight through: + +```python +nf_docs.render(pipeline, "json", indent=4) +nf_docs.render(pipeline, "yaml", default_flow_style=True) +nf_docs.render(pipeline, "html", use_tailwind=False) +nf_docs.render(pipeline, "markdown", title="My Pipeline") +``` + +Pass `single_file=True` for the focused single-document form used for modules, rather than the full +pipeline layout. + +## Writing files + +`generate()` is the equivalent of `nf-docs generate`, and returns the paths it wrote: + +```python +import nf_docs + +for path in nf_docs.generate("./my_pipeline", output_format="html", output="site/"): + print(f"wrote {path}") +``` + +Where `output` is omitted, it follows the same conventions as the CLI: `/docs/` for a +whole pipeline, and a file alongside the source when documenting a single module. + +!!! note + + `generate()` always writes files, even for `json` and `yaml` — the CLI's habit of streaming + those to stdout is a command-line convenience, not part of the API. Use `extract()` and + `render()` if you want a string. + +## Documenting a single module + +Pass a path to a `.nf` file to document just that file. A directory holding a module-style `main.nf` +(process definitions, no workflow, no pipeline config) is auto-detected, so both of these do the +same thing: + +```python +nf_docs.generate("modules/mytool/main.nf", output_format="md") +nf_docs.generate("modules/mytool", output_format="md") +``` + +Both write `modules/mytool/README.md`. + +To document every module in a repository: + +```python +from pathlib import Path +import nf_docs + +for main_nf in Path("modules").rglob("main.nf"): + nf_docs.generate(main_nf, output_format="md") +``` + +!!! warning + + Each call starts its own Language Server process, so a large loop like this is slow. Documenting + a whole pipeline in one `extract()` call is much faster than documenting its modules one by one. + +## Progress reporting + +Extraction can take a while — the Language Server has to start and index the workspace. Pass a +`progress_callback` to drive your own progress display. It receives +[`ProgressUpdate`](#progress-updates) objects. + +```python +import nf_docs + +def show(update: nf_docs.ProgressUpdate) -> None: + if update.has_progress: + print(f"{update.message} [{update.current}/{update.total}] {update.detail or ''}") + else: + print(update.message) + +pipeline = nf_docs.extract("./my_pipeline", progress_callback=show) +``` + +With `tqdm`: + +```python +from tqdm import tqdm +import nf_docs + +bar = tqdm() + +def show(update: nf_docs.ProgressUpdate) -> None: + bar.set_description(update.message) + if update.has_progress: + bar.total = update.total + bar.n = update.current + bar.refresh() + +nf_docs.extract("./my_pipeline", progress_callback=show) +``` + +`update.phase` is an `ExtractionPhase` enum member if you want to react to specific stages +(`LSP_INDEXING`, `PARSING_SCHEMA`, `COMPLETE`, and so on). + +## Configuration + +The CLI reads `~/.config/nf-docs/config.yaml`. **The Python API deliberately does not** — a library +call inside someone else's build script shouldn't change behaviour based on whoever happens to be +running it. Library calls use the defaults unless you pass a config explicitly: + +```python +import nf_docs + +# Defaults - reproducible regardless of the user's config file +nf_docs.extract("./my_pipeline") + +# Opt in to the user's config file +nf_docs.extract("./my_pipeline", config=nf_docs.load_config()) + +# Or set options directly +config = nf_docs.NfDocsConfig(ignore_config_prefixes=["genomes.", "test."]) +nf_docs.extract("./my_pipeline", config=config) +``` + +`ignore_config_prefixes` drops matching parameters from the Configuration section; it defaults to +`["genomes."]` for nf-core pipelines. Other `NfDocsConfig` fields exist but are not yet wired into +extraction — don't rely on them. + +## Caching + +Extraction results are cached in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, and a +hash of the pipeline's files. The cache is on by default, matching the CLI: + +```python +nf_docs.extract("./my_pipeline", use_cache=False) # ignore the cache entirely +nf_docs.extract("./my_pipeline", force_refresh=True) # re-extract, then update the cache + +from pathlib import Path + +from nf_docs import PipelineCache + +PipelineCache().clear(Path("./my_pipeline")) # one pipeline; takes a Path +PipelineCache().clear() # everything +``` + +## Errors + +The API raises rather than exiting. The exceptions worth catching: + +| Exception | Raised when | +| ----------------- | ------------------------------------------------------------------ | +| `ValueError` | An unsupported format, or a non-`.nf` file passed as a single file | +| `LSPError` | The Language Server can't be found, downloaded, started or queried | +| `ExtractionError` | Extraction failed | + +```python +import nf_docs + +try: + pipeline = nf_docs.extract("./my_pipeline") +except nf_docs.LSPError as e: + print(f"Language Server problem: {e}") +except nf_docs.ExtractionError as e: + print(f"Could not extract: {e}") +``` + +Missing or unparseable individual sources (no schema, a broken `nextflow.config`) are logged as +warnings and skipped, not raised. Nothing is written to the console — attach a handler to the +`nf_docs` logger if you want to see them: + +```python +import logging + +logging.getLogger("nf_docs").addHandler(logging.StreamHandler()) +logging.getLogger("nf_docs").setLevel(logging.INFO) +``` + +## The Pipeline model + +Everything is a plain `@dataclass`, so `dataclasses.asdict()`, `to_dict()`, attribute access and +`match` statements all work as you'd expect. + +| Attribute | Type | Contents | +| --------------- | --------------------- | --------------------------------------------- | +| `metadata` | `PipelineMetadata` | Name, description, version, authors, README | +| `inputs` | `list[PipelineInput]` | Typed parameters from `nextflow_schema.json` | +| `config_params` | `list[ConfigParam]` | Defaults from `nextflow.config` | +| `workflows` | `list[Workflow]` | Workflows with inputs, outputs and docstrings | +| `processes` | `list[Process]` | Processes with inputs, outputs and docstrings | +| `functions` | `list[Function]` | Functions with parameters and return values | + +Some helpers: + +```python +pipeline.has_content() # did we find anything at all? +pipeline.get_entry_workflow() # the entry workflow, or None +pipeline.get_process_by_name("FASTQC") # a Process, or None +pipeline.get_input_groups() # inputs grouped by schema section +pipeline.to_dict() # JSON-compatible dict +``` + +## Progress updates + +`ProgressUpdate` objects carry: + +| Attribute | Type | Meaning | +| -------------- | ----------------- | ------------------------------------------- | +| `phase` | `ExtractionPhase` | Which stage of extraction this is | +| `message` | `str` | Human-readable description | +| `current` | `int \| None` | Items done, when countable | +| `total` | `int \| None` | Items in total, when countable | +| `detail` | `str \| None` | Extra context, usually the current filename | +| `has_progress` | `bool` | Whether `current`/`total` are both set | +| `percent` | `float \| None` | Progress as a percentage, when countable | + +## Lower-level pieces + +The three functions above wrap classes you can use directly if you need more control: + +```python +from nf_docs import PipelineExtractor, get_renderer + +extractor = PipelineExtractor( + workspace_path="./my_pipeline", + target_file="./my_pipeline/modules/mytool/main.nf", # single-file mode + language_server_jar="/path/to/language-server-all.jar", + nextflow_path="/usr/local/bin/nextflow", +) +pipeline = extractor.extract() + +renderer = get_renderer("markdown")(title="My Pipeline") +files = renderer.render_to_directory(pipeline, "docs/") +``` + +Subclass `BaseRenderer` to add your own output format — implement `render()` and +`render_to_directory()`. + +## What's public + +Anything re-exported from the top-level `nf_docs` namespace, plus the models in `nf_docs.models`, is +the supported API. Everything else — underscore-prefixed helpers, the Language Server client +internals, the parser modules — may change without notice. + +`nf-docs` is pre-1.0, so the public API may still change between minor releases. Pin a version if +that matters to you. diff --git a/pyproject.toml b/pyproject.toml index 9a0bf6d..b3ae29d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.package-data] -nf_docs = ["templates/*"] +nf_docs = ["templates/*", "py.typed"] [tool.ruff] line-length = 100 diff --git a/src/nf_docs/__init__.py b/src/nf_docs/__init__.py index 4c9e723..f0787f9 100644 --- a/src/nf_docs/__init__.py +++ b/src/nf_docs/__init__.py @@ -3,7 +3,18 @@ This tool queries the Nextflow Language Server to extract docstrings, type information, and structure from Nextflow pipelines, producing -documentation similar to Sphinx for Python or Javadoc. +documentation similar to Sphinx for Python or Javadoc for Java. + +As well as the ``nf-docs`` command-line tool, this package is usable as a +library: + + >>> import nf_docs + >>> pipeline = nf_docs.extract("./my_pipeline") # doctest: +SKIP + >>> markdown = nf_docs.render(pipeline, "markdown") # doctest: +SKIP + +The names re-exported here, together with :mod:`nf_docs.models`, are the +supported public API. Anything else — including underscore-prefixed helpers and +the Language Server internals — may change between releases. """ try: @@ -13,11 +24,18 @@ except Exception: __version__ = "unknown" +from nf_docs.api import extract, generate, render +from nf_docs.cache import PipelineCache +from nf_docs.config import NfDocsConfig, load_config +from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root +from nf_docs.lsp_client import LSPError from nf_docs.models import ( ConfigParam, Function, + FunctionParam, Pipeline, PipelineInput, + PipelineMetadata, Process, ProcessInput, ProcessOutput, @@ -25,15 +43,53 @@ WorkflowInput, WorkflowOutput, ) +from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate +from nf_docs.renderers import ( + BaseRenderer, + HTMLRenderer, + JSONRenderer, + MarkdownRenderer, + TableRenderer, + YAMLRenderer, + get_renderer, +) __all__ = [ "__version__", + # High-level API + "extract", + "render", + "generate", + # Extraction + "PipelineExtractor", + "ExtractionError", + "LSPError", + "find_pipeline_root", + "PipelineCache", + # Configuration + "NfDocsConfig", + "load_config", + # Progress reporting + "ProgressUpdate", + "ProgressCallbackType", + "ExtractionPhase", + # Rendering + "get_renderer", + "BaseRenderer", + "HTMLRenderer", + "MarkdownRenderer", + "TableRenderer", + "JSONRenderer", + "YAMLRenderer", + # Models "Pipeline", + "PipelineMetadata", "PipelineInput", "ConfigParam", "Workflow", "Process", "Function", + "FunctionParam", "ProcessInput", "ProcessOutput", "WorkflowInput", diff --git a/src/nf_docs/api.py b/src/nf_docs/api.py new file mode 100644 index 0000000..fa37404 --- /dev/null +++ b/src/nf_docs/api.py @@ -0,0 +1,221 @@ +""" +High-level Python API for nf-docs. + +This module is the supported entry point for using nf-docs as a library rather +than a command-line tool. It wraps :class:`~nf_docs.extractor.PipelineExtractor` +and the renderers in three functions: + +- :func:`extract` — read a pipeline (or a single module) into a + :class:`~nf_docs.models.Pipeline` model +- :func:`render` — turn a ``Pipeline`` into a string in a given format +- :func:`generate` — do both and write the result to disk + +Unlike the CLI, these functions never print to the console and never exit the +process; they raise exceptions and return values instead. + +Example: + >>> import nf_docs + >>> pipeline = nf_docs.extract("./my_pipeline") # doctest: +SKIP + >>> markdown = nf_docs.render(pipeline, "markdown") # doctest: +SKIP +""" + +from pathlib import Path +from typing import Any + +from nf_docs.config import NfDocsConfig +from nf_docs.extractor import PipelineExtractor, find_pipeline_root +from nf_docs.models import Pipeline +from nf_docs.output import ( + DIRECTORY_FORMATS, + SINGLE_FILE_OUTPUT_POLICY, + normalize_format, + resolve_single_file_output, + resolve_source, +) +from nf_docs.progress import ProgressCallbackType +from nf_docs.renderers import get_renderer + +__all__ = ["extract", "render", "generate"] + + +def extract( + path: str | Path, + *, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, +) -> Pipeline: + """ + Extract documentation from a Nextflow pipeline or a single module file. + + ``path`` may be a pipeline directory or a single ``.nf`` file. A directory + containing only a module-style ``main.nf`` (process definitions, no + workflow, no pipeline config) is auto-detected as a single module — the same + rule the ``nf-docs generate`` command applies. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + config: Configuration to use. Defaults to :class:`NfDocsConfig` defaults. + Unlike the CLI, the user's ``~/.config/nf-docs/config.yaml`` is *not* + read automatically — pass ``config=load_config()`` to opt in. + language_server_jar: Path to the Nextflow Language Server JAR. Downloaded + on demand if not given. + nextflow_path: Path to the Nextflow executable + use_cache: Whether to read and write the extraction cache + force_refresh: Re-extract even when a cache entry exists + progress_callback: Called with :class:`~nf_docs.progress.ProgressUpdate` + objects as extraction proceeds + + Returns: + The extracted :class:`~nf_docs.models.Pipeline` model + + Raises: + ValueError: If ``path`` is a file that isn't a ``.nf`` file + ExtractionError: If extraction fails + LSPError: If the Language Server cannot be started or queried + """ + source, single_file_mode = resolve_source(path) + + workspace_path = find_pipeline_root(source) if single_file_mode else source + target_file = source if single_file_mode else None + + extractor = PipelineExtractor( + workspace_path=workspace_path, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + target_file=target_file, + config=config, + ) + return extractor.extract() + + +def render( + pipeline: Pipeline, + output_format: str = "html", + *, + title: str | None = None, + single_file: bool = False, + **renderer_kwargs: Any, +) -> str: + """ + Render a Pipeline model to a string. + + Args: + pipeline: The Pipeline model to render + output_format: One of ``html``, ``markdown`` (or ``md``), ``table``, + ``json``, ``yaml`` + title: Custom documentation title. Defaults to the pipeline name. + single_file: Render the focused single-document form used for modules + rather than the full pipeline form + **renderer_kwargs: Extra keyword arguments for the specific renderer, + e.g. ``use_tailwind`` (HTML), ``indent`` (JSON), + ``default_flow_style`` (YAML) + + Returns: + The rendered documentation as a string + + Raises: + ValueError: If ``output_format`` is not supported + """ + renderer = get_renderer(output_format)(title=title, **renderer_kwargs) + if single_file: + return renderer.render_single_file(pipeline) + return renderer.render(pipeline) + + +def generate( + path: str | Path, + *, + output_format: str = "html", + output: str | Path | None = None, + title: str | None = None, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, + **renderer_kwargs: Any, +) -> list[Path]: + """ + Extract a pipeline and write its documentation to disk. + + This is the programmatic equivalent of ``nf-docs generate``. Unlike the CLI, + it always writes files rather than streaming to stdout — use :func:`extract` + plus :func:`render` if you want the documentation as a string. + + When ``output`` is not given, the defaults mirror the CLI: ``/docs`` + for a whole pipeline, and a file alongside the source (e.g. ``README.md``) + when documenting a single module. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + output_format: One of ``html``, ``markdown`` (or ``md``), ``table``, + ``json``, ``yaml`` + output: Output file or directory. Defaults to the CLI's conventions. + title: Custom documentation title + config: Configuration to use (see :func:`extract`) + language_server_jar: Path to the Nextflow Language Server JAR + nextflow_path: Path to the Nextflow executable + use_cache: Whether to read and write the extraction cache + force_refresh: Re-extract even when a cache entry exists + progress_callback: Called with progress updates as extraction proceeds + **renderer_kwargs: Extra keyword arguments for the specific renderer + + Returns: + The list of files written, in the order they were created + + Raises: + ValueError: If ``path`` or ``output_format`` is invalid + ExtractionError: If extraction fails + LSPError: If the Language Server cannot be started or queried + """ + canonical_format = normalize_format(output_format) + source, single_file_mode = resolve_source(path) + + pipeline = extract( + source, + config=config, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + ) + + renderer = get_renderer(canonical_format)(title=title, **renderer_kwargs) + output_path = Path(output) if output is not None else None + + if single_file_mode: + output_file, rendered = resolve_single_file_output( + renderer, pipeline, source, canonical_format, output_path + ) + if output_file is None: + # json/yaml/table stream to stdout on the CLI; on disk they land + # next to the source file using the format's extension. + _, dir_ext = SINGLE_FILE_OUTPUT_POLICY.get(canonical_format, (None, canonical_format)) + output_file = source.parent / f"{source.stem}.{dir_ext}" + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(rendered, encoding="utf-8") + return [output_file] + + if canonical_format in DIRECTORY_FORMATS: + output_dir = output_path if output_path is not None else source / "docs" + return renderer.render_to_directory(pipeline, output_dir) + + # json / yaml → a single data file + extension = "json" if canonical_format == "json" else "yaml" + if output_path is None: + output_file = source / "docs" / f"pipeline.{extension}" + elif output_path.is_dir(): + output_file = output_path / f"pipeline.{extension}" + else: + output_file = output_path + renderer.render_to_file(pipeline, output_file) + return [output_file] diff --git a/src/nf_docs/cli.py b/src/nf_docs/cli.py index 90d41c0..bb22df1 100644 --- a/src/nf_docs/cli.py +++ b/src/nf_docs/cli.py @@ -6,7 +6,6 @@ """ import logging -import re import sys from pathlib import Path @@ -23,8 +22,10 @@ ) from nf_docs import __version__ +from nf_docs.config import load_config from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root from nf_docs.lsp_client import LSPError +from nf_docs.output import normalize_format, resolve_single_file_output, resolve_source from nf_docs.progress import ProgressUpdate from nf_docs.renderers import get_renderer @@ -167,66 +168,6 @@ def _display_path(path: Path) -> str: return str(path) -def _looks_like_module(main_nf: Path) -> bool: - """ - Heuristic: ``main.nf`` defines at least one ``process`` block and no - top-level ``workflow`` block — i.e. it's an nf-core-style module rather - than a pipeline entry point or subworkflow. - """ - try: - content = main_nf.read_text(encoding="utf-8") - except OSError: - return False - has_process = re.search(r"(?m)^\s*process\b", content) is not None - has_workflow = re.search(r"(?m)^\s*workflow\b", content) is not None - return has_process and not has_workflow - - -# Single-file output policy per format: -# default_name: the filename to use when the user passes no --output (or -# passes a directory). ``None`` means "stream to stdout instead". -# dir_ext: when streaming-default formats are forced to a file via -# ``--output some_dir/``, use ``{source_stem}.{dir_ext}``. -_SINGLE_FILE_OUTPUT_POLICY: dict[str, tuple[str | None, str]] = { - "markdown": ("README.md", "md"), - "html": ("index.html", "html"), - "json": (None, "json"), - "yaml": (None, "yaml"), - "table": (None, "txt"), -} - - -def _resolve_single_file_output( - renderer, - pipeline, - source_file: Path, - output_format: str, - output_path: Path | None, -) -> tuple[Path | None, str]: - """ - For single-file mode, decide where to write and produce the rendered string. - - Returns a tuple of ``(output_file, rendered_content)``. ``output_file`` is - ``None`` when the content should go to stdout (json/yaml/table without - explicit ``--output``). - """ - rendered = renderer.render_single_file(pipeline) - default_name, dir_ext = _SINGLE_FILE_OUTPUT_POLICY.get(output_format, (None, output_format)) - - if output_path is not None: - # User-specified path always wins. Treat directories as "put a default - # filename in here" so `-o some_dir/` still works. - if output_path.is_dir(): - filename = default_name or f"{source_file.stem}.{dir_ext}" - return output_path / filename, rendered - return output_path, rendered - - if default_name is not None: - return source_file.parent / default_name, rendered - # json / yaml / table → stdout - return None, rendered - - @main.command() @click.argument("pipeline_path", type=click.Path(exists=True, path_type=Path)) @click.option( @@ -303,53 +244,38 @@ def generate( """ setup_logging(verbose) - # Normalize format aliases - if output_format == "md": - output_format = "markdown" + output_format = normalize_format(output_format) - # Decide single-file vs directory mode up front. - single_file_mode = pipeline_path.is_file() - if single_file_mode and pipeline_path.suffix.lower() != ".nf": - console.print(f"[red]Single-file input must be a .nf file, got: {pipeline_path.name}[/red]") + # Decide single-file vs directory mode up front. A directory holding only a + # module-style main.nf is auto-detected as a single module. + try: + pipeline_path, single_file_mode = resolve_source(pipeline_path) + except ValueError as e: + console.print(f"[red]{e}[/red]") sys.exit(1) - # Auto-detect a single module: directory has a main.nf that defines - # process(es) and no workflow block, with no pipeline-level config files. - if not single_file_mode: - main_nf = pipeline_path / "main.nf" - has_pipeline_config = (pipeline_path / "nextflow.config").is_file() or ( - pipeline_path / "nextflow_schema.json" - ).is_file() - if main_nf.is_file() and not has_pipeline_config and _looks_like_module(main_nf): - pipeline_path = main_nf - single_file_mode = True - if single_file_mode: logging.getLogger("nf_docs").info("Generating single module documentation") + # The CLI honours the user's ~/.config/nf-docs/config.yaml; the Python API + # deliberately does not (see nf_docs.api.extract). + user_config = load_config() + try: with ExtractionProgressDisplay(console) as progress_display: # Extract documentation - if single_file_mode: - workspace_root = find_pipeline_root(pipeline_path) - extractor = PipelineExtractor( - workspace_path=workspace_root, - language_server_jar=language_server, - nextflow_path=nextflow_path, - use_cache=True, - force_refresh=no_cache, - progress_callback=progress_display.callback, - target_file=pipeline_path, - ) - else: - extractor = PipelineExtractor( - workspace_path=pipeline_path, - language_server_jar=language_server, - nextflow_path=nextflow_path, - use_cache=True, - force_refresh=no_cache, - progress_callback=progress_display.callback, - ) + extractor = PipelineExtractor( + workspace_path=find_pipeline_root(pipeline_path) + if single_file_mode + else pipeline_path, + language_server_jar=language_server, + nextflow_path=nextflow_path, + use_cache=True, + force_refresh=no_cache, + progress_callback=progress_display.callback, + target_file=pipeline_path if single_file_mode else None, + config=user_config, + ) pipeline = extractor.extract() @@ -382,7 +308,7 @@ def generate( renderer = renderer_class(title=title) if single_file_mode: - output_file, rendered = _resolve_single_file_output( + output_file, rendered = resolve_single_file_output( renderer, pipeline, pipeline_path, output_format, output_path ) progress.update(task, description="Rendering complete") @@ -483,6 +409,7 @@ def inspect(pipeline_path: Path, verbose: bool) -> None: extractor = PipelineExtractor( workspace_path=pipeline_path, progress_callback=progress_display.callback, + config=load_config(), ) pipeline = extractor.extract() diff --git a/src/nf_docs/extractor.py b/src/nf_docs/extractor.py index 6caa444..f8b554b 100644 --- a/src/nf_docs/extractor.py +++ b/src/nf_docs/extractor.py @@ -17,7 +17,7 @@ from typing import Any from nf_docs.cache import PipelineCache -from nf_docs.config import get_config +from nf_docs.config import NfDocsConfig from nf_docs.config_parser import parse_config from nf_docs.git_utils import GitInfo, build_source_url, get_git_info from nf_docs.lsp_client import LSPClient, SymbolKind, parse_hover_content @@ -143,6 +143,7 @@ def __init__( force_refresh: bool = False, progress_callback: ProgressCallbackType | None = None, target_file: str | Path | None = None, + config: NfDocsConfig | None = None, ): """ Initialize the extractor. @@ -157,8 +158,12 @@ def __init__( target_file: If set, extract only from this single ``.nf`` file instead of scanning the whole workspace. Pipeline-level sources (schema, config, README, cache) are skipped in this mode. + config: Configuration to use. Defaults to ``NfDocsConfig()`` defaults + rather than the user's config file, so that library callers get + reproducible results. The CLI passes ``load_config()`` explicitly. """ self.workspace_path = Path(workspace_path).resolve() + self.config = config if config is not None else NfDocsConfig() self.language_server_jar = language_server_jar self.nextflow_path = nextflow_path self.target_file = Path(target_file).resolve() if target_file else None @@ -264,7 +269,7 @@ def extract(self) -> Pipeline: pipeline.metadata = self._merge_metadata(pipeline.metadata, config_metadata) # Filter config params to exclude those already in inputs and ignored prefixes input_names = {inp.name for inp in pipeline.inputs} - config = get_config() + config = self.config pipeline.config_params = [ p for p in config_params diff --git a/src/nf_docs/output.py b/src/nf_docs/output.py new file mode 100644 index 0000000..c7382d7 --- /dev/null +++ b/src/nf_docs/output.py @@ -0,0 +1,145 @@ +""" +Output resolution policy shared by the CLI and the Python API. + +This module holds the decisions about *where* rendered documentation goes and +*what shape* it takes: format aliases, whether a path should be documented as a +single module file or as a whole pipeline, and the default filenames used when +the caller doesn't specify an output path. + +It deliberately contains no console output and no process exits so that both +``nf_docs.cli`` and ``nf_docs.api`` can share the same behaviour. +""" + +import re +from pathlib import Path + +from nf_docs.models import Pipeline +from nf_docs.renderers.base import BaseRenderer + +# Format aliases accepted on input, mapped to their canonical renderer name. +FORMAT_ALIASES: dict[str, str] = {"md": "markdown"} + +# Single-file output policy per format: +# default_name: the filename to use when the user passes no output path (or +# passes a directory). ``None`` means "stream to stdout instead". +# dir_ext: when streaming-default formats are forced to a file via an output +# directory, use ``{source_stem}.{dir_ext}``. +SINGLE_FILE_OUTPUT_POLICY: dict[str, tuple[str | None, str]] = { + "markdown": ("README.md", "md"), + "html": ("index.html", "html"), + "json": (None, "json"), + "yaml": (None, "yaml"), + "table": (None, "txt"), +} + +# Formats that render as a directory of files rather than a single document. +DIRECTORY_FORMATS: frozenset[str] = frozenset({"markdown", "html", "table"}) + + +def normalize_format(output_format: str) -> str: + """ + Normalize a format name to its canonical renderer name. + + Args: + output_format: Format name, possibly an alias (e.g. ``md``) + + Returns: + The canonical format name (e.g. ``markdown``) + """ + lowered = output_format.lower() + return FORMAT_ALIASES.get(lowered, lowered) + + +def looks_like_module(main_nf: Path) -> bool: + """ + Heuristic: ``main.nf`` defines at least one ``process`` block and no + top-level ``workflow`` block — i.e. it's an nf-core-style module rather + than a pipeline entry point or subworkflow. + + Args: + main_nf: Path to the ``main.nf`` file to inspect + + Returns: + ``True`` if the file looks like a standalone module + """ + try: + content = main_nf.read_text(encoding="utf-8") + except OSError: + return False + has_process = re.search(r"(?m)^\s*process\b", content) is not None + has_workflow = re.search(r"(?m)^\s*workflow\b", content) is not None + return has_process and not has_workflow + + +def resolve_source(path: str | Path) -> tuple[Path, bool]: + """ + Decide whether ``path`` should be documented as a single file or a pipeline. + + A ``.nf`` file is always single-file mode. A directory is auto-detected as a + single module when it holds a ``main.nf`` that defines process(es) and no + workflow block, and has no pipeline-level config files alongside it. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + + Returns: + Tuple of ``(resolved_path, single_file_mode)``. In single-file mode the + resolved path points at the ``.nf`` file itself, not its directory. + + Raises: + ValueError: If ``path`` is a file that isn't a ``.nf`` file + """ + source = Path(path) + + if source.is_file(): + if source.suffix.lower() != ".nf": + raise ValueError(f"Single-file input must be a .nf file, got: {source.name}") + return source, True + + main_nf = source / "main.nf" + has_pipeline_config = (source / "nextflow.config").is_file() or ( + source / "nextflow_schema.json" + ).is_file() + if main_nf.is_file() and not has_pipeline_config and looks_like_module(main_nf): + return main_nf, True + + return source, False + + +def resolve_single_file_output( + renderer: BaseRenderer, + pipeline: Pipeline, + source_file: Path, + output_format: str, + output_path: Path | None, +) -> tuple[Path | None, str]: + """ + For single-file mode, decide where to write and produce the rendered string. + + Args: + renderer: The renderer to produce content with + pipeline: The Pipeline model to render + source_file: The ``.nf`` file being documented + output_format: Canonical format name + output_path: Explicit output path, or ``None`` for the default + + Returns: + A tuple of ``(output_file, rendered_content)``. ``output_file`` is + ``None`` when the content should go to stdout (json/yaml/table without + an explicit output path). + """ + rendered = renderer.render_single_file(pipeline) + default_name, dir_ext = SINGLE_FILE_OUTPUT_POLICY.get(output_format, (None, output_format)) + + if output_path is not None: + # User-specified path always wins. Treat directories as "put a default + # filename in here" so `-o some_dir/` still works. + if output_path.is_dir(): + filename = default_name or f"{source_file.stem}.{dir_ext}" + return output_path / filename, rendered + return output_path, rendered + + if default_name is not None: + return source_file.parent / default_name, rendered + # json / yaml / table → stdout + return None, rendered diff --git a/src/nf_docs/py.typed b/src/nf_docs/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..e24625d --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,251 @@ +"""Tests for the high-level Python API (nf_docs.api).""" + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +import nf_docs +from nf_docs.api import extract, generate, render +from nf_docs.config import NfDocsConfig +from nf_docs.extractor import PipelineExtractor +from nf_docs.models import ConfigParam, Pipeline, PipelineMetadata, Process +from nf_docs.output import ( + looks_like_module, + normalize_format, + resolve_source, +) + + +@pytest.fixture +def module_dir(tmp_path: Path) -> Path: + """A directory holding a module-style main.nf (process, no workflow).""" + module = tmp_path / "modules" / "mytool" + module.mkdir(parents=True) + (module / "main.nf").write_text( + "process MYTOOL {\n input:\n path reads\n\n script:\n '''\n mytool\n '''\n}\n" + ) + return module + + +@pytest.fixture +def rendered_pipeline() -> Pipeline: + """A small Pipeline model that needs no extraction.""" + return Pipeline( + metadata=PipelineMetadata(name="Test Pipeline", description="A test"), + processes=[Process(name="FASTQC", docstring="Quality control")], + ) + + +class TestNormalizeFormat: + """Tests for format alias normalization.""" + + def test_md_alias(self) -> None: + assert normalize_format("md") == "markdown" + + def test_case_insensitive(self) -> None: + assert normalize_format("HTML") == "html" + + def test_passthrough(self) -> None: + assert normalize_format("json") == "json" + + +class TestResolveSource: + """Tests for single-file vs pipeline source resolution.""" + + def test_pipeline_directory(self, sample_pipeline: Path) -> None: + resolved, single_file = resolve_source(sample_pipeline) + assert resolved == sample_pipeline + assert single_file is False + + def test_nf_file(self, sample_pipeline: Path) -> None: + resolved, single_file = resolve_source(sample_pipeline / "main.nf") + assert resolved == sample_pipeline / "main.nf" + assert single_file is True + + def test_module_directory_autodetected(self, module_dir: Path) -> None: + """A bare module dir resolves to its main.nf in single-file mode.""" + resolved, single_file = resolve_source(module_dir) + assert resolved == module_dir / "main.nf" + assert single_file is True + + def test_non_nf_file_rejected(self, tmp_path: Path) -> None: + other = tmp_path / "notes.txt" + other.write_text("hello") + with pytest.raises(ValueError, match="must be a .nf file"): + resolve_source(other) + + def test_looks_like_module(self, module_dir: Path, sample_pipeline: Path) -> None: + assert looks_like_module(module_dir / "main.nf") is True + # The sample pipeline's main.nf defines a workflow, so it isn't a module + assert looks_like_module(sample_pipeline / "main.nf") is False + + +class TestExtract: + """Tests for nf_docs.extract().""" + + def test_extract_directory(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + pipeline = extract(sample_pipeline) + assert isinstance(pipeline, Pipeline) + assert pipeline.inputs + + def test_extract_single_file(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + pipeline = extract(sample_pipeline / "main.nf") + assert isinstance(pipeline, Pipeline) + + def test_rejects_non_nf_file(self, tmp_path: Path) -> None: + other = tmp_path / "notes.txt" + other.write_text("hello") + with pytest.raises(ValueError): + extract(other) + + def test_config_is_injected(self, sample_pipeline: Path) -> None: + """A passed config reaches the extractor rather than the global one.""" + config = NfDocsConfig(ignore_config_prefixes=["params."]) + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch.object( + PipelineExtractor, "extract", autospec=True, return_value=Pipeline() + ) as mock_extract: + extract(sample_pipeline, config=config) + assert mock_extract.call_args[0][0].config is config + + def test_defaults_are_hermetic(self, sample_pipeline: Path) -> None: + """Without a config argument, defaults are used - not the user's file.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch.object( + PipelineExtractor, "extract", autospec=True, return_value=Pipeline() + ) as mock_extract: + extract(sample_pipeline) + used = mock_extract.call_args[0][0].config + assert used == NfDocsConfig() + + def test_config_filters_config_params(self, sample_pipeline: Path) -> None: + """The injected config actually affects the extracted result.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch( + "nf_docs.extractor.parse_config", + return_value=(PipelineMetadata(), [ConfigParam(name="genomes.hg38")]), + ): + default = extract(sample_pipeline, use_cache=False) + permissive = extract( + sample_pipeline, + use_cache=False, + config=NfDocsConfig(ignore_config_prefixes=[]), + ) + assert [p.name for p in default.config_params] == [] + assert [p.name for p in permissive.config_params] == ["genomes.hg38"] + + def test_progress_callback_is_called(self, sample_pipeline: Path) -> None: + updates = [] + with patch.object(PipelineExtractor, "_extract_from_lsp"): + extract(sample_pipeline, use_cache=False, progress_callback=updates.append) + assert updates + assert all(hasattr(u, "phase") for u in updates) + + +class TestRender: + """Tests for nf_docs.render().""" + + @pytest.mark.parametrize("output_format", ["json", "yaml", "markdown", "html", "table"]) + def test_all_formats(self, rendered_pipeline: Pipeline, output_format: str) -> None: + result = render(rendered_pipeline, output_format) + assert isinstance(result, str) + assert result + + def test_md_alias(self, rendered_pipeline: Pipeline) -> None: + assert render(rendered_pipeline, "md") == render(rendered_pipeline, "markdown") + + def test_json_is_parseable(self, rendered_pipeline: Pipeline) -> None: + data = json.loads(render(rendered_pipeline, "json")) + assert data["pipeline"]["name"] == "Test Pipeline" + + def test_title_override(self, rendered_pipeline: Pipeline) -> None: + assert "Custom Title" in render(rendered_pipeline, "markdown", title="Custom Title") + + def test_renderer_kwargs_are_passed(self, rendered_pipeline: Pipeline) -> None: + compact = render(rendered_pipeline, "json", indent=0) + spaced = render(rendered_pipeline, "json", indent=8) + assert len(spaced) > len(compact) + + def test_single_file_mode(self, rendered_pipeline: Pipeline) -> None: + result = render(rendered_pipeline, "markdown", single_file=True) + assert isinstance(result, str) + assert result + + def test_unsupported_format(self, rendered_pipeline: Pipeline) -> None: + with pytest.raises(ValueError, match="Unsupported format"): + render(rendered_pipeline, "pdf") + + +class TestGenerate: + """Tests for nf_docs.generate().""" + + def test_returns_written_paths(self, sample_pipeline: Path, tmp_path: Path) -> None: + out = tmp_path / "site" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="html", output=out) + assert created + assert all(p.exists() for p in created) + + def test_json_to_explicit_file(self, sample_pipeline: Path, tmp_path: Path) -> None: + target = tmp_path / "api.json" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="json", output=target) + assert created == [target] + assert json.loads(target.read_text()) + + def test_json_default_location(self, sample_pipeline: Path) -> None: + """Without an output, json lands in /docs/pipeline.json.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="json") + assert created == [sample_pipeline / "docs" / "pipeline.json"] + assert created[0].exists() + + def test_markdown_default_directory(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="markdown") + assert created + assert all(p.is_relative_to(sample_pipeline / "docs") for p in created) + + def test_single_file_writes_readme(self, module_dir: Path) -> None: + """A module gets a README.md next to its main.nf.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(module_dir, output_format="md") + assert created == [module_dir / "README.md"] + assert created[0].exists() + + def test_single_file_json_writes_a_file(self, module_dir: Path) -> None: + """Unlike the CLI, generate() writes json to disk rather than stdout.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(module_dir, output_format="json") + assert created == [module_dir / "main.json"] + assert created[0].exists() + + def test_title_reaches_output(self, sample_pipeline: Path, tmp_path: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate( + sample_pipeline, + output_format="markdown", + output=tmp_path / "docs", + title="My Custom Pipeline", + ) + assert any("My Custom Pipeline" in p.read_text() for p in created) + + +class TestPublicSurface: + """The names promised in nf_docs.__all__ must resolve.""" + + def test_all_names_resolve(self) -> None: + missing = [name for name in nf_docs.__all__ if not hasattr(nf_docs, name)] + assert missing == [] + + def test_facade_is_exported(self) -> None: + assert nf_docs.extract is extract + assert nf_docs.render is render + assert nf_docs.generate is generate + + def test_py_typed_marker_ships(self) -> None: + assert (Path(nf_docs.__file__).parent / "py.typed").exists() diff --git a/zensical.toml b/zensical.toml index 93df0d6..35ef531 100644 --- a/zensical.toml +++ b/zensical.toml @@ -11,6 +11,7 @@ nav = [ {"Installation" = "installation.md"}, {"Running nf-docs" = "running.md"}, {"Writing docs" = "writing.md"}, + {"Python API" = "python-api.md"}, {"Examples" = "examples.md"}, ] From d85d0fe98cc401f19b2e52ff67a09fe623d15f9e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 19:23:26 +0000 Subject: [PATCH 2/5] docs: use rich for the progress example, and tighten the prose rich is already an nf-docs dependency, so the progress-bar example costs readers no extra install. Passing None for completed/total leaves rich's task values alone, so one callback covers both the countable and indeterminate extraction phases. Also removed em dashes, tricolons and passive constructions from the Python API page. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qwciLzAxtZMwu3uWnA24H --- README.md | 4 +- docs/python-api.md | 119 ++++++++++++++++++++++++--------------------- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 1b21312..3a861f8 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,8 @@ By default this writes `README.md` next to `main.nf`. Use `--output` to pick a d ## Use as a Python library -`nf-docs` is a CLI tool first, but it's also an importable Python package — useful for feeding -pipeline data into your own tooling or generating docs as part of a site build: +`nf-docs` is a CLI tool first, but it's also an importable Python package. Useful for feeding +pipeline data into your own tooling, or generating docs as part of a site build: ```python import nf_docs diff --git a/docs/python-api.md b/docs/python-api.md index adbf8f0..e95d956 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -1,19 +1,19 @@ # Python API -`nf-docs` is a CLI tool first, but it's a normal Python package too. If you need something the CLI -doesn't cover — feeding pipeline data into your own tooling, rendering docs as part of a site build, -or documenting modules in a loop — you can import it directly. +`nf-docs` is a CLI tool first, but it's also a normal Python package. Import it when you need +something the CLI doesn't cover: feeding pipeline data into your own tooling, or rendering docs as +part of a site build. ```bash pip install nf-docs ``` -The same requirements apply as for the CLI: Java for the Nextflow Language Server, and optionally +The requirements are the same as for the CLI: Java for the Nextflow Language Server, and optionally Nextflow itself for parsing `nextflow.config`. ## Three functions -Most things are covered by three functions: +Three functions cover most of it: | Function | Does | Returns | | ------------ | ------------------------------------------------------- | ------------ | @@ -29,13 +29,13 @@ markdown = nf_docs.render(pipeline, "markdown") files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") ``` -Unlike the CLI, these never print to the console and never exit the process — they return values and +Unlike the CLI, these never print to the console and never exit the process. They return values and raise exceptions. ## Reading pipeline data -`extract()` gives you a [`Pipeline`](#the-pipeline-model) object. This is the one to reach for when -you want the data rather than the documentation. +`extract()` gives you a [`Pipeline`](#the-pipeline-model) object. Use it when you want the data +rather than the documentation. ```python import nf_docs @@ -53,7 +53,7 @@ for param in pipeline.inputs: print(f"{param.name} ({param.type}) - {param.description}") ``` -Every model has a `to_dict()`, so handing the whole thing to something else is one call: +Every model has a `to_dict()`, so handing the whole thing to another tool is one call: ```python import json @@ -64,8 +64,8 @@ with open("pipeline-api.json", "w") as fh: ## Rendering into an existing build -`render()` returns a string, which is usually what you want when another tool owns the output -directory — a static site generator, a template, a wiki upload. +`render()` returns a string. That's usually what you want when another tool owns the output +directory, such as a static site generator. ```python from pathlib import Path @@ -78,7 +78,7 @@ Path("src/content/docs/api.md").write_text( ) ``` -Renderer-specific options are passed straight through: +Renderer-specific options pass straight through: ```python nf_docs.render(pipeline, "json", indent=4) @@ -92,7 +92,7 @@ pipeline layout. ## Writing files -`generate()` is the equivalent of `nf-docs generate`, and returns the paths it wrote: +`generate()` is the equivalent of `nf-docs generate`. It returns the paths it wrote: ```python import nf_docs @@ -101,20 +101,20 @@ for path in nf_docs.generate("./my_pipeline", output_format="html", output="site print(f"wrote {path}") ``` -Where `output` is omitted, it follows the same conventions as the CLI: `/docs/` for a -whole pipeline, and a file alongside the source when documenting a single module. +Omit `output` and it follows the same conventions as the CLI: `/docs/` for a whole +pipeline, and a file alongside the source when documenting a single module. !!! note - `generate()` always writes files, even for `json` and `yaml` — the CLI's habit of streaming - those to stdout is a command-line convenience, not part of the API. Use `extract()` and - `render()` if you want a string. + `generate()` always writes files, even for `json` and `yaml`. Streaming those to stdout is a + command-line convenience rather than part of the API. Use `extract()` and `render()` if you + want a string. ## Documenting a single module -Pass a path to a `.nf` file to document just that file. A directory holding a module-style `main.nf` -(process definitions, no workflow, no pipeline config) is auto-detected, so both of these do the -same thing: +Pass a path to a `.nf` file to document just that file. nf-docs also auto-detects a directory +holding a module-style `main.nf` (process definitions, no workflow, no pipeline config), so both of +these do the same thing: ```python nf_docs.generate("modules/mytool/main.nf", output_format="md") @@ -135,13 +135,13 @@ for main_nf in Path("modules").rglob("main.nf"): !!! warning - Each call starts its own Language Server process, so a large loop like this is slow. Documenting - a whole pipeline in one `extract()` call is much faster than documenting its modules one by one. + Each call starts its own Language Server process, so a loop like this is slow. Documenting a + whole pipeline in one `extract()` call is much faster than documenting its modules one by one. ## Progress reporting -Extraction can take a while — the Language Server has to start and index the workspace. Pass a -`progress_callback` to drive your own progress display. It receives +Extraction can take a while, because the Language Server has to start and index the workspace. Pass +a `progress_callback` to drive your own progress display. It receives [`ProgressUpdate`](#progress-updates) objects. ```python @@ -156,32 +156,41 @@ def show(update: nf_docs.ProgressUpdate) -> None: pipeline = nf_docs.extract("./my_pipeline", progress_callback=show) ``` -With `tqdm`: +`rich` ships with nf-docs, so a progress bar costs no extra dependency. Passing `None` for +`completed` or `total` leaves those values alone, so the same callback handles both the countable +and indeterminate phases: ```python -from tqdm import tqdm -import nf_docs - -bar = tqdm() +from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn -def show(update: nf_docs.ProgressUpdate) -> None: - bar.set_description(update.message) - if update.has_progress: - bar.total = update.total - bar.n = update.current - bar.refresh() +import nf_docs -nf_docs.extract("./my_pipeline", progress_callback=show) +with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), +) as progress: + task = progress.add_task("Starting...", total=None) + + def show(update: nf_docs.ProgressUpdate) -> None: + progress.update( + task, + description=update.message, + completed=update.current, + total=update.total, + ) + + pipeline = nf_docs.extract("./my_pipeline", progress_callback=show) ``` -`update.phase` is an `ExtractionPhase` enum member if you want to react to specific stages +`update.phase` is an `ExtractionPhase` member if you want to react to specific stages (`LSP_INDEXING`, `PARSING_SCHEMA`, `COMPLETE`, and so on). ## Configuration -The CLI reads `~/.config/nf-docs/config.yaml`. **The Python API deliberately does not** — a library -call inside someone else's build script shouldn't change behaviour based on whoever happens to be -running it. Library calls use the defaults unless you pass a config explicitly: +The CLI reads `~/.config/nf-docs/config.yaml`. The Python API deliberately does not: a library call +inside someone else's build script shouldn't change behaviour based on whoever happens to be running +it. Library calls use the defaults unless you pass a config explicitly: ```python import nf_docs @@ -197,14 +206,14 @@ config = nf_docs.NfDocsConfig(ignore_config_prefixes=["genomes.", "test."]) nf_docs.extract("./my_pipeline", config=config) ``` -`ignore_config_prefixes` drops matching parameters from the Configuration section; it defaults to -`["genomes."]` for nf-core pipelines. Other `NfDocsConfig` fields exist but are not yet wired into -extraction — don't rely on them. +`ignore_config_prefixes` drops matching parameters from the Configuration section, and defaults to +`["genomes."]` for nf-core pipelines. The other `NfDocsConfig` fields exist but aren't yet wired +into extraction, so don't rely on them. ## Caching -Extraction results are cached in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, and a -hash of the pipeline's files. The cache is on by default, matching the CLI: +nf-docs caches extraction results in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, +and a hash of the pipeline's files. The cache is on by default, matching the CLI: ```python nf_docs.extract("./my_pipeline", use_cache=False) # ignore the cache entirely @@ -239,9 +248,9 @@ except nf_docs.ExtractionError as e: print(f"Could not extract: {e}") ``` -Missing or unparseable individual sources (no schema, a broken `nextflow.config`) are logged as -warnings and skipped, not raised. Nothing is written to the console — attach a handler to the -`nf_docs` logger if you want to see them: +A missing or unparseable individual source (no schema, a broken `nextflow.config`) logs a warning +and carries on rather than raising. Nothing reaches the console by default. Attach a handler to the +`nf_docs` logger to see these: ```python import logging @@ -252,8 +261,8 @@ logging.getLogger("nf_docs").setLevel(logging.INFO) ## The Pipeline model -Everything is a plain `@dataclass`, so `dataclasses.asdict()`, `to_dict()`, attribute access and -`match` statements all work as you'd expect. +Everything is a plain `@dataclass`, so `to_dict()`, `dataclasses.asdict()` and ordinary attribute +access all work. | Attribute | Type | Contents | | --------------- | --------------------- | --------------------------------------------- | @@ -271,7 +280,7 @@ pipeline.has_content() # did we find anything at all? pipeline.get_entry_workflow() # the entry workflow, or None pipeline.get_process_by_name("FASTQC") # a Process, or None pipeline.get_input_groups() # inputs grouped by schema section -pipeline.to_dict() # JSON-compatible dict +pipeline.to_dict() # JSON-compatible dict ``` ## Progress updates @@ -307,14 +316,14 @@ renderer = get_renderer("markdown")(title="My Pipeline") files = renderer.render_to_directory(pipeline, "docs/") ``` -Subclass `BaseRenderer` to add your own output format — implement `render()` and +Subclass `BaseRenderer` to add your own output format, implementing `render()` and `render_to_directory()`. ## What's public Anything re-exported from the top-level `nf_docs` namespace, plus the models in `nf_docs.models`, is -the supported API. Everything else — underscore-prefixed helpers, the Language Server client -internals, the parser modules — may change without notice. +the supported API. Everything else may change without notice: underscore-prefixed helpers, the +Language Server client internals, the parser modules. `nf-docs` is pre-1.0, so the public API may still change between minor releases. Pin a version if that matters to you. From ccfa90f850b831bf25f7f4a81d0bf8ca632dc2ef Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 22:05:21 +0000 Subject: [PATCH 3/5] refactor: dedupe output policy between the CLI and the library API Cleanup pass over the library API change. No behaviour change - CLI and library output verified byte-identical across every format and output mode. output.py now owns all source/output path decisions, not just the single-file case: - resolve_data_file_path() and default_output_dir() replace the /docs and pipeline.{ext} rules that were computed separately in both api.generate() and cli.generate() - format_extension() replaces a third copy of the format->extension mapping ('json' if fmt == 'json' else 'yaml'), which silently labelled any future non-json format .yaml - cli.py now uses DIRECTORY_FORMATS instead of spelling ("markdown", "html", "table") and its inverse inline four times - resolve_single_file_output() split into a pure resolve_single_file_path() plus a streams_by_default() predicate. It no longer renders as a side effect, so the "None means stdout" sentinel is gone along with the special case in api.generate() that undid it. output.py now imports nothing from the package, keeping it a cheap leaf module. Format aliases had drifted into two tables: get_renderer() had its own "md" entry while output.FORMAT_ALIASES had another, and api.render() never normalised at all. get_renderer() now resolves through normalize_format(), so aliases live in one place. cli.generate() extracts via api.extract() rather than rebuilding the same workspace/target_file derivation, and api.generate() resolves the source once via a shared _extract_resolved() instead of resolving twice. Deleted get_config()/reset_config() and the module-level _config global. Injecting config at PipelineExtractor left them with zero callers in src/, and leaving an ambient accessor around invites reintroducing the non-hermetic behaviour the API promises against. __init__.py now resolves everything except the data models lazily via PEP 562 __getattr__. Re-exporting eagerly had pulled httpx, jinja2, markdown, pygments and yaml into every `import nf_docs`: import nf_docs 264 ms -> 90 ms (sys.modules 386 -> 151) CLI startup is unaffected either way, since cli.py already imported that whole tree; the cost fell entirely on library consumers who only wanted the models or the version. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qwciLzAxtZMwu3uWnA24H --- AGENTS.md | 21 +++-- src/nf_docs/__init__.py | 91 ++++++++++++++---- src/nf_docs/api.py | 97 ++++++++++--------- src/nf_docs/cli.py | 68 +++++++------- src/nf_docs/config.py | 25 ----- src/nf_docs/extractor.py | 3 +- src/nf_docs/output.py | 150 ++++++++++++++++++++++-------- src/nf_docs/renderers/__init__.py | 32 ++++--- tests/test_api.py | 17 +--- tests/test_config.py | 8 -- 10 files changed, 310 insertions(+), 202 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8b4b48d..63eb5bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,7 +182,7 @@ src/nf_docs/ ├── api.py # High-level library API: extract() / render() / generate() ├── cache.py # XDG-compliant caching with content hashing ├── cli.py # CLI entry point (rich-click) -├── output.py # Output path policy shared by the CLI and api.py +├── output.py # Source/output path policy shared by the CLI and api.py (leaf module) ├── config.py # User config from ~/.config/nf-docs/config.yaml ├── config_parser.py # Parse nextflow.config via `nextflow config -flat` ├── extractor.py # Main extraction orchestration @@ -218,14 +218,23 @@ src/nf_docs/ - **Rendering**: Strategy pattern via `BaseRenderer` ABC with `render()`, `render_to_file()`, `render_to_directory()`. Factory function `get_renderer(format)` returns the appropriate renderer. HTML uses a single Jinja2 template with inline CSS (Tailwind) and JavaScript for search. Adding a - new output format means updating `get_renderer()`, the CLI's `--format` choices, and tests. + new output format means updating `RENDERERS` in `renderers/__init__.py`, + `SINGLE_FILE_OUTPUT_POLICY` and (if it writes a directory) `DIRECTORY_FORMATS` in `output.py`, the + CLI's `--format` choices, and tests. Format aliases live only in `output.FORMAT_ALIASES`; + `get_renderer()` resolves through `normalize_format()` so there is one alias table. - **Config parsing**: `config_parser.py` shells out to `nextflow config -flat`. `nextflow_env.py` sets an isolated `NXF_HOME` so this doesn't pollute the user's Nextflow state. -- **Two entry points**: `cli.py` (rich-click) and `api.py` (library). Both go through `output.py` - for path/format decisions, so behaviour stays in sync. Keep console output and `sys.exit()` in +- **Two entry points**: `cli.py` (rich-click) and `api.py` (library). All source/output path and + format decisions live in `output.py`, and `cli.py` extracts via `api.extract()`, so the two stay + in sync by construction rather than by discipline. Keep console output and `sys.exit()` in `cli.py` only — core modules log via `logging` and raise exceptions. The CLI reads the user's - `~/.config/nf-docs/config.yaml` and passes it in explicitly; `api.py` uses `NfDocsConfig()` - defaults so library calls are reproducible. + `~/.config/nf-docs/config.yaml` and passes it to `PipelineExtractor(config=...)`; `api.py` uses + `NfDocsConfig()` defaults so library calls are reproducible. There is deliberately no global + config accessor — inject an `NfDocsConfig` instead of reintroducing ambient state. +- **Lazy exports**: `__init__.py` eagerly imports only `models`; everything else in `__all__` is + resolved on first access via PEP 562 `__getattr__`. This keeps `import nf_docs` at ~90 ms instead + of ~260 ms for callers that only want the models. Add new public names to both `_LAZY_EXPORTS` and + the `TYPE_CHECKING` block so type checkers and IDEs still see them. - **Public API**: everything in `nf_docs.__all__` plus `nf_docs.models`. Adding to it is a commitment — update `docs/python-api.md` and `tests/test_api.py` alongside. - **Pre-commit hook entry**: `.pre-commit-hooks.yaml` exposes the CLI as the `nf-docs` hook with diff --git a/src/nf_docs/__init__.py b/src/nf_docs/__init__.py index f0787f9..dbf2f19 100644 --- a/src/nf_docs/__init__.py +++ b/src/nf_docs/__init__.py @@ -13,10 +13,16 @@ >>> markdown = nf_docs.render(pipeline, "markdown") # doctest: +SKIP The names re-exported here, together with :mod:`nf_docs.models`, are the -supported public API. Anything else — including underscore-prefixed helpers and -the Language Server internals — may change between releases. +supported public API. Anything else - including underscore-prefixed helpers and +the Language Server internals - may change between releases. + +Everything except the data models is resolved lazily (PEP 562), so +``import nf_docs`` stays cheap for callers that only want the models or the +version. Touching any other name pulls in its module on first access. """ +from typing import TYPE_CHECKING, Any + try: from importlib.metadata import version @@ -24,11 +30,8 @@ except Exception: __version__ = "unknown" -from nf_docs.api import extract, generate, render -from nf_docs.cache import PipelineCache -from nf_docs.config import NfDocsConfig, load_config -from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root -from nf_docs.lsp_client import LSPError +# The models are cheap (dataclasses and stdlib only) and are what most callers +# reach for first, so they stay eager. from nf_docs.models import ( ConfigParam, Function, @@ -43,16 +46,70 @@ WorkflowInput, WorkflowOutput, ) -from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate -from nf_docs.renderers import ( - BaseRenderer, - HTMLRenderer, - JSONRenderer, - MarkdownRenderer, - TableRenderer, - YAMLRenderer, - get_renderer, -) + +if TYPE_CHECKING: + # Imported for type checkers and IDEs only; at runtime __getattr__ below + # resolves these on first use. + from nf_docs.api import extract, generate, render + from nf_docs.cache import PipelineCache + from nf_docs.config import NfDocsConfig, load_config + from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root + from nf_docs.lsp_client import LSPError + from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate + from nf_docs.renderers import ( + BaseRenderer, + HTMLRenderer, + JSONRenderer, + MarkdownRenderer, + TableRenderer, + YAMLRenderer, + get_renderer, + ) + +# Public name -> module that defines it. Pulling in nf_docs.renderers costs +# jinja2, markdown and pygments; nf_docs.lsp_client costs httpx. Callers that +# only want the models shouldn't pay for either. +_LAZY_EXPORTS: dict[str, str] = { + "extract": "nf_docs.api", + "render": "nf_docs.api", + "generate": "nf_docs.api", + "PipelineCache": "nf_docs.cache", + "NfDocsConfig": "nf_docs.config", + "load_config": "nf_docs.config", + "ExtractionError": "nf_docs.extractor", + "PipelineExtractor": "nf_docs.extractor", + "find_pipeline_root": "nf_docs.extractor", + "LSPError": "nf_docs.lsp_client", + "ExtractionPhase": "nf_docs.progress", + "ProgressCallbackType": "nf_docs.progress", + "ProgressUpdate": "nf_docs.progress", + "BaseRenderer": "nf_docs.renderers", + "HTMLRenderer": "nf_docs.renderers", + "JSONRenderer": "nf_docs.renderers", + "MarkdownRenderer": "nf_docs.renderers", + "TableRenderer": "nf_docs.renderers", + "YAMLRenderer": "nf_docs.renderers", + "get_renderer": "nf_docs.renderers", +} + + +def __getattr__(name: str) -> Any: + """Resolve a lazily-exported name on first access (PEP 562).""" + module_name = _LAZY_EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + import importlib + + value = getattr(importlib.import_module(module_name), name) + globals()[name] = value # cache, so __getattr__ runs once per name + return value + + +def __dir__() -> list[str]: + """Include the lazy exports in dir() and tab completion.""" + return sorted({*globals(), *_LAZY_EXPORTS}) + __all__ = [ "__version__", diff --git a/src/nf_docs/api.py b/src/nf_docs/api.py index fa37404..d23dad4 100644 --- a/src/nf_docs/api.py +++ b/src/nf_docs/api.py @@ -27,9 +27,10 @@ from nf_docs.models import Pipeline from nf_docs.output import ( DIRECTORY_FORMATS, - SINGLE_FILE_OUTPUT_POLICY, + default_output_dir, normalize_format, - resolve_single_file_output, + resolve_data_file_path, + resolve_single_file_path, resolve_source, ) from nf_docs.progress import ProgressCallbackType @@ -38,6 +39,40 @@ __all__ = ["extract", "render", "generate"] +def _extract_resolved( + source: Path, + single_file_mode: bool, + *, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, +) -> Pipeline: + """ + Extract from an already-resolved source, so resolution happens exactly once. + + Args: + source: Resolved path, as returned by ``resolve_source`` + single_file_mode: Whether ``source`` is a single ``.nf`` file + + Returns: + The extracted Pipeline model + """ + extractor = PipelineExtractor( + workspace_path=find_pipeline_root(source) if single_file_mode else source, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + target_file=source if single_file_mode else None, + config=config, + ) + return extractor.extract() + + def extract( path: str | Path, *, @@ -78,21 +113,16 @@ def extract( LSPError: If the Language Server cannot be started or queried """ source, single_file_mode = resolve_source(path) - - workspace_path = find_pipeline_root(source) if single_file_mode else source - target_file = source if single_file_mode else None - - extractor = PipelineExtractor( - workspace_path=workspace_path, + return _extract_resolved( + source, + single_file_mode, + config=config, language_server_jar=language_server_jar, nextflow_path=nextflow_path, use_cache=use_cache, force_refresh=force_refresh, progress_callback=progress_callback, - target_file=target_file, - config=config, ) - return extractor.extract() def render( @@ -123,7 +153,7 @@ def render( Raises: ValueError: If ``output_format`` is not supported """ - renderer = get_renderer(output_format)(title=title, **renderer_kwargs) + renderer = get_renderer(normalize_format(output_format))(title=title, **renderer_kwargs) if single_file: return renderer.render_single_file(pipeline) return renderer.render(pipeline) @@ -160,12 +190,8 @@ def generate( ``json``, ``yaml`` output: Output file or directory. Defaults to the CLI's conventions. title: Custom documentation title - config: Configuration to use (see :func:`extract`) - language_server_jar: Path to the Nextflow Language Server JAR - nextflow_path: Path to the Nextflow executable - use_cache: Whether to read and write the extraction cache - force_refresh: Re-extract even when a cache entry exists - progress_callback: Called with progress updates as extraction proceeds + config, language_server_jar, nextflow_path, use_cache, force_refresh, + progress_callback: See :func:`extract`. **renderer_kwargs: Extra keyword arguments for the specific renderer Returns: @@ -177,10 +203,13 @@ def generate( LSPError: If the Language Server cannot be started or queried """ canonical_format = normalize_format(output_format) + renderer = get_renderer(canonical_format)(title=title, **renderer_kwargs) source, single_file_mode = resolve_source(path) + output_path = Path(output) if output is not None else None - pipeline = extract( + pipeline = _extract_resolved( source, + single_file_mode, config=config, language_server_jar=language_server_jar, nextflow_path=nextflow_path, @@ -189,33 +218,17 @@ def generate( progress_callback=progress_callback, ) - renderer = get_renderer(canonical_format)(title=title, **renderer_kwargs) - output_path = Path(output) if output is not None else None - if single_file_mode: - output_file, rendered = resolve_single_file_output( - renderer, pipeline, source, canonical_format, output_path - ) - if output_file is None: - # json/yaml/table stream to stdout on the CLI; on disk they land - # next to the source file using the format's extension. - _, dir_ext = SINGLE_FILE_OUTPUT_POLICY.get(canonical_format, (None, canonical_format)) - output_file = source.parent / f"{source.stem}.{dir_ext}" + output_file = resolve_single_file_path(source, canonical_format, output_path) output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.write_text(rendered, encoding="utf-8") + output_file.write_text(renderer.render_single_file(pipeline), encoding="utf-8") return [output_file] if canonical_format in DIRECTORY_FORMATS: - output_dir = output_path if output_path is not None else source / "docs" - return renderer.render_to_directory(pipeline, output_dir) - - # json / yaml → a single data file - extension = "json" if canonical_format == "json" else "yaml" - if output_path is None: - output_file = source / "docs" / f"pipeline.{extension}" - elif output_path.is_dir(): - output_file = output_path / f"pipeline.{extension}" - else: - output_file = output_path + return renderer.render_to_directory( + pipeline, output_path if output_path is not None else default_output_dir(source) + ) + + output_file = resolve_data_file_path(source, canonical_format, output_path) renderer.render_to_file(pipeline, output_file) return [output_file] diff --git a/src/nf_docs/cli.py b/src/nf_docs/cli.py index bb22df1..c5b947e 100644 --- a/src/nf_docs/cli.py +++ b/src/nf_docs/cli.py @@ -22,10 +22,19 @@ ) from nf_docs import __version__ +from nf_docs.api import extract as extract_pipeline from nf_docs.config import load_config -from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root +from nf_docs.extractor import ExtractionError, PipelineExtractor from nf_docs.lsp_client import LSPError -from nf_docs.output import normalize_format, resolve_single_file_output, resolve_source +from nf_docs.output import ( + DIRECTORY_FORMATS, + default_output_dir, + normalize_format, + resolve_data_file_path, + resolve_single_file_path, + resolve_source, + streams_by_default, +) from nf_docs.progress import ProgressUpdate from nf_docs.renderers import get_renderer @@ -263,22 +272,15 @@ def generate( try: with ExtractionProgressDisplay(console) as progress_display: - # Extract documentation - extractor = PipelineExtractor( - workspace_path=find_pipeline_root(pipeline_path) - if single_file_mode - else pipeline_path, + pipeline = extract_pipeline( + pipeline_path, + config=user_config, language_server_jar=language_server, nextflow_path=nextflow_path, - use_cache=True, force_refresh=no_cache, progress_callback=progress_display.callback, - target_file=pipeline_path if single_file_mode else None, - config=user_config, ) - pipeline = extractor.extract() - # Check if any content was found if not pipeline.has_content(): if single_file_mode: @@ -308,35 +310,31 @@ def generate( renderer = renderer_class(title=title) if single_file_mode: - output_file, rendered = resolve_single_file_output( - renderer, pipeline, pipeline_path, output_format, output_path + rendered = renderer.render_single_file(pipeline) + # Formats with no natural single-file name go to stdout unless + # the user asked for a specific output path. + output_file = ( + None + if output_path is None and streams_by_default(output_format) + else resolve_single_file_path(pipeline_path, output_format, output_path) ) progress.update(task, description="Rendering complete") elif output_path: # Write to file/directory - if output_format in ("markdown", "html", "table"): + if output_format in DIRECTORY_FORMATS: created_files = renderer.render_to_directory(pipeline, output_path) - progress.update(task, description="Rendering complete") else: - # JSON/YAML - write to single file - # If output_path is a directory, use default filename - if output_path.is_dir(): - ext = "json" if output_format == "json" else "yaml" - output_file = output_path / f"pipeline.{ext}" - else: - output_file = output_path - output_file.parent.mkdir(parents=True, exist_ok=True) + # JSON/YAML - write to a single data file + output_file = resolve_data_file_path(pipeline_path, output_format, output_path) renderer.render_to_file(pipeline, output_file) - progress.update(task, description="Rendering complete") + progress.update(task, description="Rendering complete") else: # Write to stdout or default directory - if output_format in ("json", "yaml"): - pass # Will write to stdout after progress - else: - # Write to default directory - default_dir = pipeline_path / "docs" + if output_format in DIRECTORY_FORMATS: + default_dir = default_output_dir(pipeline_path) created_files = renderer.render_to_directory(pipeline, default_dir) progress.update(task, description="Rendering complete") + # json/yaml stream to stdout after the progress display is gone # Output results after progress display is gone if single_file_mode: @@ -351,7 +349,7 @@ def generate( ) console.print(f" - {_display_path(output_file)}") elif output_path: - if output_format in ("markdown", "html", "table"): + if output_format in DIRECTORY_FORMATS: file_word = "file" if len(created_files) == 1 else "files" console.print( f"[green]Created {len(created_files)} {file_word} in {output_path}[/green]" @@ -361,16 +359,16 @@ def generate( else: console.print(f"[green]Written to {output_file}[/green]") else: - if output_format in ("json", "yaml"): - # Write to stdout - click.echo(renderer.render(pipeline)) - else: + if output_format in DIRECTORY_FORMATS: file_word = "file" if len(created_files) == 1 else "files" console.print( f"[green]Created {len(created_files)} {file_word} in {default_dir}[/green]" ) for f in created_files: console.print(f" - ./{_display_path(f)}") + else: + # Write to stdout + click.echo(renderer.render(pipeline)) except LSPError as e: console.print(f"[red]Language Server error: {e}[/red]") diff --git a/src/nf_docs/config.py b/src/nf_docs/config.py index 3282ff7..d0196c7 100644 --- a/src/nf_docs/config.py +++ b/src/nf_docs/config.py @@ -217,28 +217,3 @@ def get_example_config() -> str: # - "tests/**/*.nf" # - "examples/**/*.nf" """ - - -# Global config instance (loaded lazily) -_config: NfDocsConfig | None = None - - -def get_config() -> NfDocsConfig: - """ - Get the global configuration instance. - - Loads the config from file on first call, then returns cached instance. - - Returns: - The global NfDocsConfig instance. - """ - global _config - if _config is None: - _config = load_config() - return _config - - -def reset_config() -> None: - """Reset the global config instance (useful for testing).""" - global _config - _config = None diff --git a/src/nf_docs/extractor.py b/src/nf_docs/extractor.py index f8b554b..c38e308 100644 --- a/src/nf_docs/extractor.py +++ b/src/nf_docs/extractor.py @@ -269,11 +269,10 @@ def extract(self) -> Pipeline: pipeline.metadata = self._merge_metadata(pipeline.metadata, config_metadata) # Filter config params to exclude those already in inputs and ignored prefixes input_names = {inp.name for inp in pipeline.inputs} - config = self.config pipeline.config_params = [ p for p in config_params - if p.name not in input_names and not config.should_ignore_config_param(p.name) + if p.name not in input_names and not self.config.should_ignore_config_param(p.name) ] except Exception as e: logger.warning(f"Failed to parse config: {e}") diff --git a/src/nf_docs/output.py b/src/nf_docs/output.py index c7382d7..3c861c2 100644 --- a/src/nf_docs/output.py +++ b/src/nf_docs/output.py @@ -1,29 +1,29 @@ """ -Output resolution policy shared by the CLI and the Python API. +Source and output resolution shared by the CLI and the Python API. -This module holds the decisions about *where* rendered documentation goes and -*what shape* it takes: format aliases, whether a path should be documented as a -single module file or as a whole pipeline, and the default filenames used when -the caller doesn't specify an output path. +This module answers two questions for both entry points: -It deliberately contains no console output and no process exits so that both -``nf_docs.cli`` and ``nf_docs.api`` can share the same behaviour. +- what is being documented (a whole pipeline, or a single ``.nf`` module) +- where the rendered documentation goes on disk + +It holds no console output, no process exits, and no rendering. Keeping the +answers here is what stops ``nf_docs.cli`` and ``nf_docs.api`` from drifting +apart. It imports nothing from the rest of the package, so it stays cheap. """ import re from pathlib import Path -from nf_docs.models import Pipeline -from nf_docs.renderers.base import BaseRenderer - # Format aliases accepted on input, mapped to their canonical renderer name. +# ``renderers.get_renderer()`` resolves through this too, so a new alias only +# needs adding here. FORMAT_ALIASES: dict[str, str] = {"md": "markdown"} -# Single-file output policy per format: -# default_name: the filename to use when the user passes no output path (or -# passes a directory). ``None`` means "stream to stdout instead". -# dir_ext: when streaming-default formats are forced to a file via an output -# directory, use ``{source_stem}.{dir_ext}``. +# Per-format single-file output policy: +# default_name: filename to use when no output path is given. ``None`` means +# the format has no natural single-file name, so the CLI streams it to +# stdout instead (see ``resolve_single_file_path``). +# extension: this format's file extension. SINGLE_FILE_OUTPUT_POLICY: dict[str, tuple[str | None, str]] = { "markdown": ("README.md", "md"), "html": ("index.html", "html"), @@ -35,6 +35,12 @@ # Formats that render as a directory of files rather than a single document. DIRECTORY_FORMATS: frozenset[str] = frozenset({"markdown", "html", "table"}) +# Filename stem used for json/yaml output of a whole pipeline. +DATA_FILE_STEM = "pipeline" + +# Directory used when documenting a whole pipeline with no explicit output path. +DEFAULT_OUTPUT_DIR = "docs" + def normalize_format(output_format: str) -> str: """ @@ -50,6 +56,22 @@ def normalize_format(output_format: str) -> str: return FORMAT_ALIASES.get(lowered, lowered) +def format_extension(output_format: str) -> str: + """ + Get the file extension for a canonical format name. + + Args: + output_format: Canonical format name + + Returns: + The extension, without a leading dot + + Raises: + KeyError: If the format has no output policy entry + """ + return SINGLE_FILE_OUTPUT_POLICY[output_format][1] + + def looks_like_module(main_nf: Path) -> bool: """ Heuristic: ``main.nf`` defines at least one ``process`` block and no @@ -79,6 +101,9 @@ def resolve_source(path: str | Path) -> tuple[Path, bool]: single module when it holds a ``main.nf`` that defines process(es) and no workflow block, and has no pipeline-level config files alongside it. + Resolving an already-resolved path is a no-op, so callers that pass the + result back in get the same answer. + Args: path: Path to a Nextflow pipeline directory or a single ``.nf`` file @@ -106,40 +131,85 @@ def resolve_source(path: str | Path) -> tuple[Path, bool]: return source, False -def resolve_single_file_output( - renderer: BaseRenderer, - pipeline: Pipeline, +def streams_by_default(output_format: str) -> bool: + """ + Whether a format has no natural single-file name, so the CLI streams it to + stdout when given no output path. + + This is a command-line concern: the Python API always writes a file. + + Args: + output_format: Canonical format name + + Returns: + ``True`` if the CLI streams this format rather than writing a file + + Raises: + KeyError: If the format has no output policy entry + """ + return SINGLE_FILE_OUTPUT_POLICY[output_format][0] is None + + +def resolve_single_file_path( source_file: Path, output_format: str, output_path: Path | None, -) -> tuple[Path | None, str]: +) -> Path: """ - For single-file mode, decide where to write and produce the rendered string. + Decide which file a single module's documentation is written to. Args: - renderer: The renderer to produce content with - pipeline: The Pipeline model to render source_file: The ``.nf`` file being documented output_format: Canonical format name output_path: Explicit output path, or ``None`` for the default Returns: - A tuple of ``(output_file, rendered_content)``. ``output_file`` is - ``None`` when the content should go to stdout (json/yaml/table without - an explicit output path). + The file to write + + Raises: + KeyError: If the format has no output policy entry """ - rendered = renderer.render_single_file(pipeline) - default_name, dir_ext = SINGLE_FILE_OUTPUT_POLICY.get(output_format, (None, output_format)) - - if output_path is not None: - # User-specified path always wins. Treat directories as "put a default - # filename in here" so `-o some_dir/` still works. - if output_path.is_dir(): - filename = default_name or f"{source_file.stem}.{dir_ext}" - return output_path / filename, rendered - return output_path, rendered - - if default_name is not None: - return source_file.parent / default_name, rendered - # json / yaml / table → stdout - return None, rendered + default_name, extension = SINGLE_FILE_OUTPUT_POLICY[output_format] + filename = default_name or f"{source_file.stem}.{extension}" + + if output_path is None: + return source_file.parent / filename + # User-specified path always wins. Treat directories as "put a default + # filename in here" so `-o some_dir/` still works. + if output_path.is_dir(): + return output_path / filename + return output_path + + +def resolve_data_file_path( + source: Path, + output_format: str, + output_path: Path | None, +) -> Path: + """ + Decide where a whole pipeline's json/yaml output goes. + + Args: + source: The pipeline directory being documented + output_format: Canonical format name (``json`` or ``yaml``) + output_path: Explicit output file or directory, or ``None`` for the + default of ``/docs/pipeline.`` + + Returns: + The file to write + + Raises: + KeyError: If the format has no output policy entry + """ + filename = f"{DATA_FILE_STEM}.{format_extension(output_format)}" + + if output_path is None: + return source / DEFAULT_OUTPUT_DIR / filename + if output_path.is_dir(): + return output_path / filename + return output_path + + +def default_output_dir(source: Path) -> Path: + """Get the default output directory for a whole pipeline.""" + return source / DEFAULT_OUTPUT_DIR diff --git a/src/nf_docs/renderers/__init__.py b/src/nf_docs/renderers/__init__.py index 1270db0..95a7216 100644 --- a/src/nf_docs/renderers/__init__.py +++ b/src/nf_docs/renderers/__init__.py @@ -9,6 +9,7 @@ Table: Compact Markdown tables (terraform-docs style) """ +from nf_docs.output import FORMAT_ALIASES, normalize_format from nf_docs.renderers.base import BaseRenderer from nf_docs.renderers.html import HTMLRenderer from nf_docs.renderers.json import JSONRenderer @@ -23,16 +24,28 @@ "YAMLRenderer", "MarkdownRenderer", "HTMLRenderer", + "RENDERERS", "get_renderer", ] +# Canonical format name -> renderer class. Aliases (e.g. "md") are resolved by +# ``nf_docs.output.normalize_format`` before lookup, so they live in one place. +RENDERERS: dict[str, type[BaseRenderer]] = { + "json": JSONRenderer, + "yaml": YAMLRenderer, + "markdown": MarkdownRenderer, + "html": HTMLRenderer, + "table": TableRenderer, +} + + def get_renderer(format: str) -> type[BaseRenderer]: """ Get the renderer class for a given format. Args: - format: Output format (json, yaml, markdown, html) + format: Output format (json, yaml, markdown/md, html, table) Returns: Renderer class @@ -40,18 +53,9 @@ def get_renderer(format: str) -> type[BaseRenderer]: Raises: ValueError: If format is not supported """ - renderers = { - "json": JSONRenderer, - "yaml": YAMLRenderer, - "markdown": MarkdownRenderer, - "md": MarkdownRenderer, - "html": HTMLRenderer, - "table": TableRenderer, - } - - format_lower = format.lower() - if format_lower not in renderers: - supported = ", ".join(sorted(set(renderers.keys()))) + canonical = normalize_format(format) + if canonical not in RENDERERS: + supported = ", ".join(sorted({*RENDERERS, *FORMAT_ALIASES})) raise ValueError(f"Unsupported format: {format}. Supported formats: {supported}") - return renderers[format_lower] + return RENDERERS[canonical] diff --git a/tests/test_api.py b/tests/test_api.py index e24625d..ed2920f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -105,22 +105,13 @@ def test_rejects_non_nf_file(self, tmp_path: Path) -> None: def test_config_is_injected(self, sample_pipeline: Path) -> None: """A passed config reaches the extractor rather than the global one.""" config = NfDocsConfig(ignore_config_prefixes=["params."]) - with patch.object(PipelineExtractor, "_extract_from_lsp"): - with patch.object( - PipelineExtractor, "extract", autospec=True, return_value=Pipeline() - ) as mock_extract: - extract(sample_pipeline, config=config) - assert mock_extract.call_args[0][0].config is config + with patch("nf_docs.api.PipelineExtractor") as mock_cls: + extract(sample_pipeline, config=config) + assert mock_cls.call_args.kwargs["config"] is config def test_defaults_are_hermetic(self, sample_pipeline: Path) -> None: """Without a config argument, defaults are used - not the user's file.""" - with patch.object(PipelineExtractor, "_extract_from_lsp"): - with patch.object( - PipelineExtractor, "extract", autospec=True, return_value=Pipeline() - ) as mock_extract: - extract(sample_pipeline) - used = mock_extract.call_args[0][0].config - assert used == NfDocsConfig() + assert PipelineExtractor(workspace_path=sample_pipeline).config == NfDocsConfig() def test_config_filters_config_params(self, sample_pipeline: Path) -> None: """The injected config actually affects the extracted result.""" diff --git a/tests/test_config.py b/tests/test_config.py index 8c6fcdc..4d46e72 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,7 +11,6 @@ get_example_config, get_xdg_config_home, load_config, - reset_config, ) @@ -147,13 +146,6 @@ def test_to_dict(self) -> None: class TestLoadConfig: """Tests for load_config function.""" - @pytest.fixture(autouse=True) - def reset(self) -> None: - """Reset global config before and after each test.""" - reset_config() - yield - reset_config() - def test_load_default_when_file_missing(self, tmp_path: Path) -> None: """Test that defaults are used when config file doesn't exist.""" config = load_config(tmp_path / "nonexistent" / "config.yaml") From 560ccf45d230f4b6dc438f4c844e81468cd07fbe Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 07:37:29 +0000 Subject: [PATCH 4/5] refactor: go back to plain eager re-exports in __init__.py The lazy PEP 562 __getattr__ saved ~170 ms on `import nf_docs`, but that is not worth the maintenance cost: every new public name had to be registered in both _LAZY_EXPORTS and a TYPE_CHECKING block, and the mechanism is non-obvious to anyone adding one. nf-docs is not a library you import in a hot path. It is imported to parse pipeline docs, which takes seconds waiting on the Language Server, so the import time is noise. A plain list of imports is easier to read and harder to get wrong. The rest of the cleanup pass stands - only __init__.py reverts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qwciLzAxtZMwu3uWnA24H --- AGENTS.md | 9 ++--- src/nf_docs/__init__.py | 87 +++++++---------------------------------- 2 files changed, 19 insertions(+), 77 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 63eb5bf..1f1bcea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -231,12 +231,11 @@ src/nf_docs/ `~/.config/nf-docs/config.yaml` and passes it to `PipelineExtractor(config=...)`; `api.py` uses `NfDocsConfig()` defaults so library calls are reproducible. There is deliberately no global config accessor — inject an `NfDocsConfig` instead of reintroducing ambient state. -- **Lazy exports**: `__init__.py` eagerly imports only `models`; everything else in `__all__` is - resolved on first access via PEP 562 `__getattr__`. This keeps `import nf_docs` at ~90 ms instead - of ~260 ms for callers that only want the models. Add new public names to both `_LAZY_EXPORTS` and - the `TYPE_CHECKING` block so type checkers and IDEs still see them. - **Public API**: everything in `nf_docs.__all__` plus `nf_docs.models`. Adding to it is a - commitment — update `docs/python-api.md` and `tests/test_api.py` alongside. + commitment — update `docs/python-api.md` and `tests/test_api.py` alongside. `__init__.py` + re-exports eagerly, which costs ~170 ms on `import nf_docs` (it pulls in jinja2, httpx and + friends). That's deliberate: a plain import list is worth more than the startup time for a tool + that spends seconds waiting on the Language Server. Don't make it lazy. - **Pre-commit hook entry**: `.pre-commit-hooks.yaml` exposes the CLI as the `nf-docs` hook with default `args: [., --format, html]`, triggered by changes to `.nf`, `nextflow.config`, `nextflow_schema.json`, `meta.yml`, or README files. Guarded by `tests/test_pre_commit_hooks.py`. diff --git a/src/nf_docs/__init__.py b/src/nf_docs/__init__.py index dbf2f19..4b2b14f 100644 --- a/src/nf_docs/__init__.py +++ b/src/nf_docs/__init__.py @@ -15,14 +15,8 @@ The names re-exported here, together with :mod:`nf_docs.models`, are the supported public API. Anything else - including underscore-prefixed helpers and the Language Server internals - may change between releases. - -Everything except the data models is resolved lazily (PEP 562), so -``import nf_docs`` stays cheap for callers that only want the models or the -version. Touching any other name pulls in its module on first access. """ -from typing import TYPE_CHECKING, Any - try: from importlib.metadata import version @@ -30,8 +24,11 @@ except Exception: __version__ = "unknown" -# The models are cheap (dataclasses and stdlib only) and are what most callers -# reach for first, so they stay eager. +from nf_docs.api import extract, generate, render +from nf_docs.cache import PipelineCache +from nf_docs.config import NfDocsConfig, load_config +from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root +from nf_docs.lsp_client import LSPError from nf_docs.models import ( ConfigParam, Function, @@ -46,70 +43,16 @@ WorkflowInput, WorkflowOutput, ) - -if TYPE_CHECKING: - # Imported for type checkers and IDEs only; at runtime __getattr__ below - # resolves these on first use. - from nf_docs.api import extract, generate, render - from nf_docs.cache import PipelineCache - from nf_docs.config import NfDocsConfig, load_config - from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root - from nf_docs.lsp_client import LSPError - from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate - from nf_docs.renderers import ( - BaseRenderer, - HTMLRenderer, - JSONRenderer, - MarkdownRenderer, - TableRenderer, - YAMLRenderer, - get_renderer, - ) - -# Public name -> module that defines it. Pulling in nf_docs.renderers costs -# jinja2, markdown and pygments; nf_docs.lsp_client costs httpx. Callers that -# only want the models shouldn't pay for either. -_LAZY_EXPORTS: dict[str, str] = { - "extract": "nf_docs.api", - "render": "nf_docs.api", - "generate": "nf_docs.api", - "PipelineCache": "nf_docs.cache", - "NfDocsConfig": "nf_docs.config", - "load_config": "nf_docs.config", - "ExtractionError": "nf_docs.extractor", - "PipelineExtractor": "nf_docs.extractor", - "find_pipeline_root": "nf_docs.extractor", - "LSPError": "nf_docs.lsp_client", - "ExtractionPhase": "nf_docs.progress", - "ProgressCallbackType": "nf_docs.progress", - "ProgressUpdate": "nf_docs.progress", - "BaseRenderer": "nf_docs.renderers", - "HTMLRenderer": "nf_docs.renderers", - "JSONRenderer": "nf_docs.renderers", - "MarkdownRenderer": "nf_docs.renderers", - "TableRenderer": "nf_docs.renderers", - "YAMLRenderer": "nf_docs.renderers", - "get_renderer": "nf_docs.renderers", -} - - -def __getattr__(name: str) -> Any: - """Resolve a lazily-exported name on first access (PEP 562).""" - module_name = _LAZY_EXPORTS.get(name) - if module_name is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - import importlib - - value = getattr(importlib.import_module(module_name), name) - globals()[name] = value # cache, so __getattr__ runs once per name - return value - - -def __dir__() -> list[str]: - """Include the lazy exports in dir() and tab completion.""" - return sorted({*globals(), *_LAZY_EXPORTS}) - +from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate +from nf_docs.renderers import ( + BaseRenderer, + HTMLRenderer, + JSONRenderer, + MarkdownRenderer, + TableRenderer, + YAMLRenderer, + get_renderer, +) __all__ = [ "__version__", From 104f74aa9fd2216518aa44d616ed0981709a034e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 08:15:16 +0000 Subject: [PATCH 5/5] fix: address code review findings on the library API Four issues, all introduced by this PR, each reproduced before and after. The cache ignored configuration. Its key was version + workspace path + file contents, so a CLI run (which loads the user's config file) and a library call (which uses defaults) shared an entry for the same unchanged pipeline and returned each other's config-filtered results. Reproduced in both directions. NfDocsConfig.cache_key() now folds into the content hash, and PipelineCache.get/set take the config. Folding into the existing hash rather than adding a filename component keeps _cleanup_old_caches working unchanged. The trade-off is that alternating two configs on one pipeline evicts each entry in turn instead of keeping both - a re-extraction, where sharing an entry gave wrong answers. load_config() had moved outside the CLI's try block, so it no longer sat inside the extractor's `except Exception: logger.warning(...)`. A config file holding valid YAML of the wrong shape (a top-level list) escaped as an uncaught AttributeError and dumped a traceback; `inspect` was unaffected, only `generate` regressed. load_config() now checks the parsed value is a mapping, which fixes it for every caller rather than just that call site. resolve_source() didn't check the path exists. The CLI gets that from click's exists=True, but the new public API had nothing, so extract("./my_pipline") returned an empty Pipeline named after the typo with no error at all. (It did not start the Language Server - there's an early return on an empty file scan.) Tests wrote into the developer's real ~/.cache/nf-docs/: 32 entries per pytest run, containing empty extractions because the Language Server is mocked. An autouse conftest fixture now points XDG_CACHE_HOME and XDG_CONFIG_HOME at tmp dirs. Regression tests added for all four. CLI and library output re-verified byte-identical across every format and output mode. Not addressed: resolve_single_file_path() treats a not-yet-created output directory as a file, because is_dir() is false for it. That predates this PR - the same check was in the original _resolve_single_file_output - so it belongs in its own change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qwciLzAxtZMwu3uWnA24H --- CHANGELOG.md | 10 +++++++++ docs/python-api.md | 16 +++++++------- src/nf_docs/cache.py | 45 +++++++++++++++++++++++++++++++++------- src/nf_docs/config.py | 26 +++++++++++++++++++++++ src/nf_docs/extractor.py | 13 +++++++++--- src/nf_docs/output.py | 9 +++++++- tests/conftest.py | 14 +++++++++++++ tests/test_api.py | 26 +++++++++++++++++++++++ tests/test_config.py | 14 +++++++++++++ 9 files changed, 155 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1b6bf..3aa61c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ of relying on the global instance - New "Python API" documentation page +### Fixed + +- The extraction cache is now keyed on the configuration as well as the pipeline contents. + Previously a CLI run (which loads `~/.config/nf-docs/config.yaml`) and a library call (which uses + defaults) shared a cache entry for the same unchanged pipeline and returned each other's results +- A config file containing valid YAML of the wrong shape (e.g. a top-level list) falls back to + defaults with a warning instead of raising +- `nf_docs.extract()` and `nf_docs.generate()` raise `ValueError` for a path that doesn't exist, + rather than returning an empty `Pipeline` named after the missing directory + ### Changed - Output path policy (format aliases, single-module detection, default filenames) moved from diff --git a/docs/python-api.md b/docs/python-api.md index e95d956..2649ad1 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -212,8 +212,10 @@ into extraction, so don't rely on them. ## Caching -nf-docs caches extraction results in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, -and a hash of the pipeline's files. The cache is on by default, matching the CLI: +nf-docs caches extraction results in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, a +hash of the pipeline's files, and the configuration used. The config is part of the key so a CLI run +and a library call on the same pipeline don't return each other's results. The cache is on by +default, matching the CLI: ```python nf_docs.extract("./my_pipeline", use_cache=False) # ignore the cache entirely @@ -231,11 +233,11 @@ PipelineCache().clear() # everything The API raises rather than exiting. The exceptions worth catching: -| Exception | Raised when | -| ----------------- | ------------------------------------------------------------------ | -| `ValueError` | An unsupported format, or a non-`.nf` file passed as a single file | -| `LSPError` | The Language Server can't be found, downloaded, started or queried | -| `ExtractionError` | Extraction failed | +| Exception | Raised when | +| ----------------- | --------------------------------------------------------------------------------------------- | +| `ValueError` | A path that doesn't exist, an unsupported format, or a non-`.nf` file passed as a single file | +| `LSPError` | The Language Server can't be found, downloaded, started or queried | +| `ExtractionError` | Extraction failed | ```python import nf_docs diff --git a/src/nf_docs/cache.py b/src/nf_docs/cache.py index 946f539..7b4b367 100644 --- a/src/nf_docs/cache.py +++ b/src/nf_docs/cache.py @@ -13,6 +13,7 @@ from typing import Any import nf_docs +from nf_docs.config import NfDocsConfig from nf_docs.models import ( ConfigParam, Function, @@ -133,17 +134,41 @@ def _get_target_file_hash(self, target_file: Path) -> str: logger.debug(f"Could not hash {meta}: {e}") return hasher.hexdigest()[:32] - def _get_cache_path(self, workspace: Path, target_file: Path | None = None) -> Path: - """Get the cache file path for a workspace (or a target file within it).""" + def _get_cache_path( + self, + workspace: Path, + target_file: Path | None = None, + config: NfDocsConfig | None = None, + ) -> Path: + """ + Get the cache file path for a workspace (or a target file within it). + + The configuration is folded into the content hash rather than added as a + separate filename component, which keeps ``_cleanup_old_caches`` working + unchanged. The trade-off is that alternating between two configs on one + pipeline evicts each entry in turn instead of keeping both; that costs a + re-extraction, where sharing an entry would return wrong results. + """ workspace_hash = self._get_workspace_hash(workspace) + config_hash = (config or NfDocsConfig()).cache_key() if target_file is not None: - target_hash = self._get_target_file_hash(target_file) + target_hash = self._combine(self._get_target_file_hash(target_file), config_hash) return self.cache_dir / f"{nf_docs.__version__}_{workspace_hash}_mod_{target_hash}.json" - content_hash = self._get_content_hash(workspace) + content_hash = self._combine(self._get_content_hash(workspace), config_hash) # Include version in filename to invalidate old caches when nf-docs is updated return self.cache_dir / f"{nf_docs.__version__}_{workspace_hash}_{content_hash}.json" - def get(self, workspace: Path, target_file: Path | None = None) -> Pipeline | None: + @staticmethod + def _combine(content_hash: str, config_hash: str) -> str: + """Fold a config hash into a content hash, keeping the digest length.""" + return hashlib.sha256(f"{content_hash}:{config_hash}".encode()).hexdigest() + + def get( + self, + workspace: Path, + target_file: Path | None = None, + config: NfDocsConfig | None = None, + ) -> Pipeline | None: """ Get cached Pipeline if valid. @@ -152,11 +177,14 @@ def get(self, workspace: Path, target_file: Path | None = None) -> Pipeline | No target_file: For single-file mode, the .nf file being extracted. The cache key becomes per-file so different modules within the same workspace don't collide. + config: Configuration the result was extracted with. Entries are + only reused for a matching config; defaults are assumed when + this is not given. Returns: Cached Pipeline if valid, None if cache miss or stale """ - cache_path = self._get_cache_path(workspace, target_file) + cache_path = self._get_cache_path(workspace, target_file, config) if not cache_path.exists(): logger.debug("Cache miss: no cache file") @@ -177,6 +205,7 @@ def set( workspace: Path, pipeline: Pipeline, target_file: Path | None = None, + config: NfDocsConfig | None = None, ) -> None: """ Store Pipeline in cache. @@ -185,10 +214,12 @@ def set( workspace: Path to the pipeline workspace pipeline: Pipeline model to cache target_file: For single-file mode, the .nf file being extracted. + config: Configuration the result was extracted with, so that a + different config doesn't read this entry back. """ self.cache_dir.mkdir(parents=True, exist_ok=True) - cache_path = self._get_cache_path(workspace, target_file) + cache_path = self._get_cache_path(workspace, target_file, config) # Clean up old cache files (per-workspace for pipeline mode, per-file # for single-file mode — we don't want one module to evict another). diff --git a/src/nf_docs/config.py b/src/nf_docs/config.py index d0196c7..af1ade0 100644 --- a/src/nf_docs/config.py +++ b/src/nf_docs/config.py @@ -8,6 +8,8 @@ if no config file exists. """ +import hashlib +import json import logging import os from dataclasses import dataclass, field @@ -116,6 +118,21 @@ def from_dict(cls, data: dict[str, Any]) -> "NfDocsConfig": exclude_patterns=data.get("exclude_patterns", DEFAULT_CONFIG["exclude_patterns"]), ) + def cache_key(self) -> str: + """ + Stable short hash of this configuration. + + Extraction results depend on the config in use, so the cache key has to + include it. Without this, a CLI run (which loads the user's config file) + and a library call (which uses defaults) would share a cache entry on + the same unchanged pipeline and return each other's results. + + Returns: + A hex digest suitable for use in a cache filename. + """ + payload = json.dumps(self.to_dict(), sort_keys=True) + return hashlib.sha256(payload.encode()).hexdigest()[:16] + def to_dict(self) -> dict[str, Any]: """Convert config to a dictionary.""" return { @@ -151,6 +168,15 @@ def load_config(config_path: Path | None = None) -> NfDocsConfig: with open(config_path, encoding="utf-8") as f: data = yaml.safe_load(f) or {} + if not isinstance(data, dict): + # Valid YAML, wrong shape (e.g. a top-level list). Treat it the same + # as unparseable rather than letting an AttributeError escape. + logger.warning( + f"Config file {config_path} must contain a mapping of settings, " + f"got {type(data).__name__}. Using defaults." + ) + return NfDocsConfig() + logger.debug(f"Loaded config from {config_path}") return NfDocsConfig.from_dict(data) diff --git a/src/nf_docs/extractor.py b/src/nf_docs/extractor.py index c38e308..d6bc1aa 100644 --- a/src/nf_docs/extractor.py +++ b/src/nf_docs/extractor.py @@ -202,7 +202,9 @@ def extract(self) -> Pipeline: message="Checking cache...", ) ) - cached = self.cache.get(self.workspace_path, target_file=self.target_file) + cached = self.cache.get( + self.workspace_path, target_file=self.target_file, config=self.config + ) if cached: self._progress( ProgressUpdate( @@ -223,7 +225,12 @@ def extract(self) -> Pipeline: self._extract_from_lsp(pipeline, git_info) if self.cache: - self.cache.set(self.workspace_path, pipeline, target_file=self.target_file) + self.cache.set( + self.workspace_path, + pipeline, + target_file=self.target_file, + config=self.config, + ) self._progress( ProgressUpdate( @@ -314,7 +321,7 @@ def extract(self) -> Pipeline: # Store in cache if self.cache: - self.cache.set(self.workspace_path, pipeline) + self.cache.set(self.workspace_path, pipeline, config=self.config) self._progress( ProgressUpdate( diff --git a/src/nf_docs/output.py b/src/nf_docs/output.py index 3c861c2..e5946b6 100644 --- a/src/nf_docs/output.py +++ b/src/nf_docs/output.py @@ -112,10 +112,17 @@ def resolve_source(path: str | Path) -> tuple[Path, bool]: resolved path points at the ``.nf`` file itself, not its directory. Raises: - ValueError: If ``path`` is a file that isn't a ``.nf`` file + ValueError: If ``path`` does not exist, or is a file that isn't a + ``.nf`` file """ source = Path(path) + if not source.exists(): + # The CLI gets this from click's exists=True; library callers don't, and + # extracting a typo'd path otherwise yields an empty Pipeline named + # after the typo rather than an error. + raise ValueError(f"Path does not exist: {source}") + if source.is_file(): if source.suffix.lower() != ".nf": raise ValueError(f"Single-file input must be a .nf file, got: {source.name}") diff --git a/tests/conftest.py b/tests/conftest.py index 1b01333..de1afd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,20 @@ import pytest +@pytest.fixture(autouse=True) +def isolate_xdg_dirs(tmp_path_factory, monkeypatch) -> None: + """ + Point XDG cache and config at throwaway directories for every test. + + Without this, any test that extracts with caching left entries in the + developer's real ``~/.cache/nf-docs/`` - and because the Language Server is + mocked out in tests, those entries are empty results that a later real run + could pick up. + """ + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path_factory.mktemp("xdg_cache"))) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path_factory.mktemp("xdg_config"))) + + @pytest.fixture def sample_schema() -> dict: """Sample nextflow_schema.json content.""" diff --git a/tests/test_api.py b/tests/test_api.py index ed2920f..cf854ec 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -102,6 +102,32 @@ def test_rejects_non_nf_file(self, tmp_path: Path) -> None: with pytest.raises(ValueError): extract(other) + def test_rejects_missing_path(self, tmp_path: Path) -> None: + """A typo'd path must raise, not return an empty Pipeline named after it.""" + with pytest.raises(ValueError, match="does not exist"): + extract(tmp_path / "no_such_pipeline") + + def test_cache_is_keyed_on_config(self, sample_pipeline: Path) -> None: + """ + Two configs must not share a cache entry. + + Without the config in the cache key a CLI run (user config) and a + library call (defaults) return each other's results for an unchanged + pipeline. + """ + params = (PipelineMetadata(), [ConfigParam(name="genomes.mm10")]) + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch("nf_docs.extractor.parse_config", return_value=params): + # Caching left ON deliberately - that is the thing under test. + permissive = extract( + sample_pipeline, config=NfDocsConfig(ignore_config_prefixes=[]) + ) + restrictive = extract( + sample_pipeline, config=NfDocsConfig(ignore_config_prefixes=["genomes."]) + ) + assert [p.name for p in permissive.config_params] == ["genomes.mm10"] + assert [p.name for p in restrictive.config_params] == [] + def test_config_is_injected(self, sample_pipeline: Path) -> None: """A passed config reaches the extractor rather than the global one.""" config = NfDocsConfig(ignore_config_prefixes=["params."]) diff --git a/tests/test_config.py b/tests/test_config.py index 4d46e72..126ac95 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -153,6 +153,20 @@ def test_load_default_when_file_missing(self, tmp_path: Path) -> None: assert config.ignore_config_prefixes == ["genomes."] assert config.include_hidden_params is False + @pytest.mark.parametrize( + "contents", + ["- ignore_config_prefixes\n- genomes.\n", "just a string\n", "42\n"], + ids=["list", "string", "number"], + ) + def test_load_default_when_yaml_is_not_a_mapping(self, tmp_path: Path, contents: str) -> None: + """Valid YAML of the wrong shape falls back to defaults, it doesn't raise.""" + config_file = tmp_path / "config.yaml" + config_file.write_text(contents) + + config = load_config(config_file) + + assert config == NfDocsConfig() + def test_load_from_file(self, tmp_path: Path) -> None: """Test loading config from a valid YAML file.""" config_file = tmp_path / "config.yaml"