diff --git a/AGENTS.md b/AGENTS.md index 1728e58..1f1bcea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -178,9 +178,11 @@ Async test mode is `auto` (via `asyncio_mode = "auto"` in pyproject.toml). ``` src/nf_docs/ -├── __init__.py # Version from importlib.metadata, re-exports models +├── __init__.py # Version, public API re-exports (__all__) +├── api.py # High-level library API: extract() / render() / generate() ├── cache.py # XDG-compliant caching with content hashing ├── cli.py # CLI entry point (rich-click) +├── output.py # Source/output path policy shared by the CLI and api.py (leaf module) ├── config.py # User config from ~/.config/nf-docs/config.yaml ├── config_parser.py # Parse nextflow.config via `nextflow config -flat` ├── extractor.py # Main extraction orchestration @@ -216,9 +218,24 @@ src/nf_docs/ - **Rendering**: Strategy pattern via `BaseRenderer` ABC with `render()`, `render_to_file()`, `render_to_directory()`. Factory function `get_renderer(format)` returns the appropriate renderer. HTML uses a single Jinja2 template with inline CSS (Tailwind) and JavaScript for search. Adding a - new output format means updating `get_renderer()`, the CLI's `--format` choices, and tests. + new output format means updating `RENDERERS` in `renderers/__init__.py`, + `SINGLE_FILE_OUTPUT_POLICY` and (if it writes a directory) `DIRECTORY_FORMATS` in `output.py`, the + CLI's `--format` choices, and tests. Format aliases live only in `output.FORMAT_ALIASES`; + `get_renderer()` resolves through `normalize_format()` so there is one alias table. - **Config parsing**: `config_parser.py` shells out to `nextflow config -flat`. `nextflow_env.py` sets an isolated `NXF_HOME` so this doesn't pollute the user's Nextflow state. +- **Two entry points**: `cli.py` (rich-click) and `api.py` (library). All source/output path and + format decisions live in `output.py`, and `cli.py` extracts via `api.extract()`, so the two stay + in sync by construction rather than by discipline. Keep console output and `sys.exit()` in + `cli.py` only — core modules log via `logging` and raise exceptions. The CLI reads the user's + `~/.config/nf-docs/config.yaml` and passes it to `PipelineExtractor(config=...)`; `api.py` uses + `NfDocsConfig()` defaults so library calls are reproducible. There is deliberately no global + config accessor — inject an `NfDocsConfig` instead of reintroducing ambient state. +- **Public API**: everything in `nf_docs.__all__` plus `nf_docs.models`. Adding to it is a + commitment — update `docs/python-api.md` and `tests/test_api.py` alongside. `__init__.py` + re-exports eagerly, which costs ~170 ms on `import nf_docs` (it pulls in jinja2, httpx and + friends). That's deliberate: a plain import list is worth more than the startup time for a tool + that spends seconds waiting on the Language Server. Don't make it lazy. - **Pre-commit hook entry**: `.pre-commit-hooks.yaml` exposes the CLI as the `nf-docs` hook with default `args: [., --format, html]`, triggered by changes to `.nf`, `nextflow.config`, `nextflow_schema.json`, `meta.yml`, or README files. Guarded by `tests/test_pre_commit_hooks.py`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fe9c27..3aa61c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## Unreleased + +### Added + +- Supported Python API for using nf-docs as a library: `nf_docs.extract()`, `nf_docs.render()` and + `nf_docs.generate()`, alongside a wider set of exports from the top-level `nf_docs` namespace +- `py.typed` marker so type hints reach downstream users +- `PipelineExtractor` accepts a `config=` argument, so callers can supply an `NfDocsConfig` instead + of relying on the global instance +- New "Python API" documentation page + +### Fixed + +- The extraction cache is now keyed on the configuration as well as the pipeline contents. + Previously a CLI run (which loads `~/.config/nf-docs/config.yaml`) and a library call (which uses + defaults) shared a cache entry for the same unchanged pipeline and returned each other's results +- A config file containing valid YAML of the wrong shape (e.g. a top-level list) falls back to + defaults with a warning instead of raising +- `nf_docs.extract()` and `nf_docs.generate()` raise `ValueError` for a path that doesn't exist, + rather than returning an empty `Pipeline` named after the missing directory + +### Changed + +- Output path policy (format aliases, single-module detection, default filenames) moved from + `cli.py` into a new `nf_docs.output` module so the CLI and the Python API share one implementation +- Library callers now get `NfDocsConfig()` defaults rather than the user's + `~/.config/nf-docs/config.yaml`, which the CLI continues to read. This keeps programmatic use + reproducible. + ## [Version 0.4.0](https://github.com/ewels/nf-docs/releases/tag/v0.4.0) - 2026-05-29 ### Added diff --git a/MANIFEST.in b/MANIFEST.in index ff8bc3d..07edd72 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ recursive-include src/nf_docs/templates *.html *.css +include src/nf_docs/py.typed diff --git a/README.md b/README.md index 008a108..3a861f8 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,23 @@ nf-docs generate modules/mytool/subtools/main.nf --format md By default this writes `README.md` next to `main.nf`. Use `--output` to pick a different path, or `--format json` / `--format yaml` to print structured output to stdout instead. +## Use as a Python library + +`nf-docs` is a CLI tool first, but it's also an importable Python package. Useful for feeding +pipeline data into your own tooling, or generating docs as part of a site build: + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") +print([p.name for p in pipeline.processes]) + +markdown = nf_docs.render(pipeline, "markdown") +files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") +``` + +See the [Python API docs](https://ewels.github.io/nf-docs/python-api/) for the full reference. + ## Prek / pre-commit hook `nf-docs` can generate pipeline documentation automatically before each commit using diff --git a/docs/python-api.md b/docs/python-api.md new file mode 100644 index 0000000..2649ad1 --- /dev/null +++ b/docs/python-api.md @@ -0,0 +1,331 @@ +# Python API + +`nf-docs` is a CLI tool first, but it's also a normal Python package. Import it when you need +something the CLI doesn't cover: feeding pipeline data into your own tooling, or rendering docs as +part of a site build. + +```bash +pip install nf-docs +``` + +The requirements are the same as for the CLI: Java for the Nextflow Language Server, and optionally +Nextflow itself for parsing `nextflow.config`. + +## Three functions + +Three functions cover most of it: + +| Function | Does | Returns | +| ------------ | ------------------------------------------------------- | ------------ | +| `extract()` | Reads a pipeline (or a single module) into a data model | `Pipeline` | +| `render()` | Turns a `Pipeline` into a string in a given format | `str` | +| `generate()` | Both of the above, then writes the result to disk | `list[Path]` | + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") +markdown = nf_docs.render(pipeline, "markdown") +files = nf_docs.generate("./my_pipeline", output_format="html", output="site/") +``` + +Unlike the CLI, these never print to the console and never exit the process. They return values and +raise exceptions. + +## Reading pipeline data + +`extract()` gives you a [`Pipeline`](#the-pipeline-model) object. Use it when you want the data +rather than the documentation. + +```python +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") + +print(pipeline.metadata.name, pipeline.metadata.version) + +for process in pipeline.processes: + print(f"{process.name}: {process.docstring}") + +# Required parameters that have no default +for param in pipeline.inputs: + if param.required and param.default is None: + print(f"{param.name} ({param.type}) - {param.description}") +``` + +Every model has a `to_dict()`, so handing the whole thing to another tool is one call: + +```python +import json + +with open("pipeline-api.json", "w") as fh: + json.dump(pipeline.to_dict(), fh, indent=2) +``` + +## Rendering into an existing build + +`render()` returns a string. That's usually what you want when another tool owns the output +directory, such as a static site generator. + +```python +from pathlib import Path +import nf_docs + +pipeline = nf_docs.extract("./my_pipeline") + +Path("src/content/docs/api.md").write_text( + "---\ntitle: Pipeline API\n---\n\n" + nf_docs.render(pipeline, "markdown") +) +``` + +Renderer-specific options pass straight through: + +```python +nf_docs.render(pipeline, "json", indent=4) +nf_docs.render(pipeline, "yaml", default_flow_style=True) +nf_docs.render(pipeline, "html", use_tailwind=False) +nf_docs.render(pipeline, "markdown", title="My Pipeline") +``` + +Pass `single_file=True` for the focused single-document form used for modules, rather than the full +pipeline layout. + +## Writing files + +`generate()` is the equivalent of `nf-docs generate`. It returns the paths it wrote: + +```python +import nf_docs + +for path in nf_docs.generate("./my_pipeline", output_format="html", output="site/"): + print(f"wrote {path}") +``` + +Omit `output` and it follows the same conventions as the CLI: `/docs/` for a whole +pipeline, and a file alongside the source when documenting a single module. + +!!! note + + `generate()` always writes files, even for `json` and `yaml`. Streaming those to stdout is a + command-line convenience rather than part of the API. Use `extract()` and `render()` if you + want a string. + +## Documenting a single module + +Pass a path to a `.nf` file to document just that file. nf-docs also auto-detects a directory +holding a module-style `main.nf` (process definitions, no workflow, no pipeline config), so both of +these do the same thing: + +```python +nf_docs.generate("modules/mytool/main.nf", output_format="md") +nf_docs.generate("modules/mytool", output_format="md") +``` + +Both write `modules/mytool/README.md`. + +To document every module in a repository: + +```python +from pathlib import Path +import nf_docs + +for main_nf in Path("modules").rglob("main.nf"): + nf_docs.generate(main_nf, output_format="md") +``` + +!!! warning + + Each call starts its own Language Server process, so a loop like this is slow. Documenting a + whole pipeline in one `extract()` call is much faster than documenting its modules one by one. + +## Progress reporting + +Extraction can take a while, because the Language Server has to start and index the workspace. Pass +a `progress_callback` to drive your own progress display. It receives +[`ProgressUpdate`](#progress-updates) objects. + +```python +import nf_docs + +def show(update: nf_docs.ProgressUpdate) -> None: + if update.has_progress: + print(f"{update.message} [{update.current}/{update.total}] {update.detail or ''}") + else: + print(update.message) + +pipeline = nf_docs.extract("./my_pipeline", progress_callback=show) +``` + +`rich` ships with nf-docs, so a progress bar costs no extra dependency. Passing `None` for +`completed` or `total` leaves those values alone, so the same callback handles both the countable +and indeterminate phases: + +```python +from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn + +import nf_docs + +with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), +) as progress: + task = progress.add_task("Starting...", total=None) + + def show(update: nf_docs.ProgressUpdate) -> None: + progress.update( + task, + description=update.message, + completed=update.current, + total=update.total, + ) + + pipeline = nf_docs.extract("./my_pipeline", progress_callback=show) +``` + +`update.phase` is an `ExtractionPhase` member if you want to react to specific stages +(`LSP_INDEXING`, `PARSING_SCHEMA`, `COMPLETE`, and so on). + +## Configuration + +The CLI reads `~/.config/nf-docs/config.yaml`. The Python API deliberately does not: a library call +inside someone else's build script shouldn't change behaviour based on whoever happens to be running +it. Library calls use the defaults unless you pass a config explicitly: + +```python +import nf_docs + +# Defaults - reproducible regardless of the user's config file +nf_docs.extract("./my_pipeline") + +# Opt in to the user's config file +nf_docs.extract("./my_pipeline", config=nf_docs.load_config()) + +# Or set options directly +config = nf_docs.NfDocsConfig(ignore_config_prefixes=["genomes.", "test."]) +nf_docs.extract("./my_pipeline", config=config) +``` + +`ignore_config_prefixes` drops matching parameters from the Configuration section, and defaults to +`["genomes."]` for nf-core pipelines. The other `NfDocsConfig` fields exist but aren't yet wired +into extraction, so don't rely on them. + +## Caching + +nf-docs caches extraction results in `~/.cache/nf-docs/`, keyed by pipeline path, nf-docs version, a +hash of the pipeline's files, and the configuration used. The config is part of the key so a CLI run +and a library call on the same pipeline don't return each other's results. The cache is on by +default, matching the CLI: + +```python +nf_docs.extract("./my_pipeline", use_cache=False) # ignore the cache entirely +nf_docs.extract("./my_pipeline", force_refresh=True) # re-extract, then update the cache + +from pathlib import Path + +from nf_docs import PipelineCache + +PipelineCache().clear(Path("./my_pipeline")) # one pipeline; takes a Path +PipelineCache().clear() # everything +``` + +## Errors + +The API raises rather than exiting. The exceptions worth catching: + +| Exception | Raised when | +| ----------------- | --------------------------------------------------------------------------------------------- | +| `ValueError` | A path that doesn't exist, an unsupported format, or a non-`.nf` file passed as a single file | +| `LSPError` | The Language Server can't be found, downloaded, started or queried | +| `ExtractionError` | Extraction failed | + +```python +import nf_docs + +try: + pipeline = nf_docs.extract("./my_pipeline") +except nf_docs.LSPError as e: + print(f"Language Server problem: {e}") +except nf_docs.ExtractionError as e: + print(f"Could not extract: {e}") +``` + +A missing or unparseable individual source (no schema, a broken `nextflow.config`) logs a warning +and carries on rather than raising. Nothing reaches the console by default. Attach a handler to the +`nf_docs` logger to see these: + +```python +import logging + +logging.getLogger("nf_docs").addHandler(logging.StreamHandler()) +logging.getLogger("nf_docs").setLevel(logging.INFO) +``` + +## The Pipeline model + +Everything is a plain `@dataclass`, so `to_dict()`, `dataclasses.asdict()` and ordinary attribute +access all work. + +| Attribute | Type | Contents | +| --------------- | --------------------- | --------------------------------------------- | +| `metadata` | `PipelineMetadata` | Name, description, version, authors, README | +| `inputs` | `list[PipelineInput]` | Typed parameters from `nextflow_schema.json` | +| `config_params` | `list[ConfigParam]` | Defaults from `nextflow.config` | +| `workflows` | `list[Workflow]` | Workflows with inputs, outputs and docstrings | +| `processes` | `list[Process]` | Processes with inputs, outputs and docstrings | +| `functions` | `list[Function]` | Functions with parameters and return values | + +Some helpers: + +```python +pipeline.has_content() # did we find anything at all? +pipeline.get_entry_workflow() # the entry workflow, or None +pipeline.get_process_by_name("FASTQC") # a Process, or None +pipeline.get_input_groups() # inputs grouped by schema section +pipeline.to_dict() # JSON-compatible dict +``` + +## Progress updates + +`ProgressUpdate` objects carry: + +| Attribute | Type | Meaning | +| -------------- | ----------------- | ------------------------------------------- | +| `phase` | `ExtractionPhase` | Which stage of extraction this is | +| `message` | `str` | Human-readable description | +| `current` | `int \| None` | Items done, when countable | +| `total` | `int \| None` | Items in total, when countable | +| `detail` | `str \| None` | Extra context, usually the current filename | +| `has_progress` | `bool` | Whether `current`/`total` are both set | +| `percent` | `float \| None` | Progress as a percentage, when countable | + +## Lower-level pieces + +The three functions above wrap classes you can use directly if you need more control: + +```python +from nf_docs import PipelineExtractor, get_renderer + +extractor = PipelineExtractor( + workspace_path="./my_pipeline", + target_file="./my_pipeline/modules/mytool/main.nf", # single-file mode + language_server_jar="/path/to/language-server-all.jar", + nextflow_path="/usr/local/bin/nextflow", +) +pipeline = extractor.extract() + +renderer = get_renderer("markdown")(title="My Pipeline") +files = renderer.render_to_directory(pipeline, "docs/") +``` + +Subclass `BaseRenderer` to add your own output format, implementing `render()` and +`render_to_directory()`. + +## What's public + +Anything re-exported from the top-level `nf_docs` namespace, plus the models in `nf_docs.models`, is +the supported API. Everything else may change without notice: underscore-prefixed helpers, the +Language Server client internals, the parser modules. + +`nf-docs` is pre-1.0, so the public API may still change between minor releases. Pin a version if +that matters to you. diff --git a/pyproject.toml b/pyproject.toml index 9a0bf6d..b3ae29d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.package-data] -nf_docs = ["templates/*"] +nf_docs = ["templates/*", "py.typed"] [tool.ruff] line-length = 100 diff --git a/src/nf_docs/__init__.py b/src/nf_docs/__init__.py index 4c9e723..4b2b14f 100644 --- a/src/nf_docs/__init__.py +++ b/src/nf_docs/__init__.py @@ -3,7 +3,18 @@ This tool queries the Nextflow Language Server to extract docstrings, type information, and structure from Nextflow pipelines, producing -documentation similar to Sphinx for Python or Javadoc. +documentation similar to Sphinx for Python or Javadoc for Java. + +As well as the ``nf-docs`` command-line tool, this package is usable as a +library: + + >>> import nf_docs + >>> pipeline = nf_docs.extract("./my_pipeline") # doctest: +SKIP + >>> markdown = nf_docs.render(pipeline, "markdown") # doctest: +SKIP + +The names re-exported here, together with :mod:`nf_docs.models`, are the +supported public API. Anything else - including underscore-prefixed helpers and +the Language Server internals - may change between releases. """ try: @@ -13,11 +24,18 @@ except Exception: __version__ = "unknown" +from nf_docs.api import extract, generate, render +from nf_docs.cache import PipelineCache +from nf_docs.config import NfDocsConfig, load_config +from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root +from nf_docs.lsp_client import LSPError from nf_docs.models import ( ConfigParam, Function, + FunctionParam, Pipeline, PipelineInput, + PipelineMetadata, Process, ProcessInput, ProcessOutput, @@ -25,15 +43,53 @@ WorkflowInput, WorkflowOutput, ) +from nf_docs.progress import ExtractionPhase, ProgressCallbackType, ProgressUpdate +from nf_docs.renderers import ( + BaseRenderer, + HTMLRenderer, + JSONRenderer, + MarkdownRenderer, + TableRenderer, + YAMLRenderer, + get_renderer, +) __all__ = [ "__version__", + # High-level API + "extract", + "render", + "generate", + # Extraction + "PipelineExtractor", + "ExtractionError", + "LSPError", + "find_pipeline_root", + "PipelineCache", + # Configuration + "NfDocsConfig", + "load_config", + # Progress reporting + "ProgressUpdate", + "ProgressCallbackType", + "ExtractionPhase", + # Rendering + "get_renderer", + "BaseRenderer", + "HTMLRenderer", + "MarkdownRenderer", + "TableRenderer", + "JSONRenderer", + "YAMLRenderer", + # Models "Pipeline", + "PipelineMetadata", "PipelineInput", "ConfigParam", "Workflow", "Process", "Function", + "FunctionParam", "ProcessInput", "ProcessOutput", "WorkflowInput", diff --git a/src/nf_docs/api.py b/src/nf_docs/api.py new file mode 100644 index 0000000..d23dad4 --- /dev/null +++ b/src/nf_docs/api.py @@ -0,0 +1,234 @@ +""" +High-level Python API for nf-docs. + +This module is the supported entry point for using nf-docs as a library rather +than a command-line tool. It wraps :class:`~nf_docs.extractor.PipelineExtractor` +and the renderers in three functions: + +- :func:`extract` — read a pipeline (or a single module) into a + :class:`~nf_docs.models.Pipeline` model +- :func:`render` — turn a ``Pipeline`` into a string in a given format +- :func:`generate` — do both and write the result to disk + +Unlike the CLI, these functions never print to the console and never exit the +process; they raise exceptions and return values instead. + +Example: + >>> import nf_docs + >>> pipeline = nf_docs.extract("./my_pipeline") # doctest: +SKIP + >>> markdown = nf_docs.render(pipeline, "markdown") # doctest: +SKIP +""" + +from pathlib import Path +from typing import Any + +from nf_docs.config import NfDocsConfig +from nf_docs.extractor import PipelineExtractor, find_pipeline_root +from nf_docs.models import Pipeline +from nf_docs.output import ( + DIRECTORY_FORMATS, + default_output_dir, + normalize_format, + resolve_data_file_path, + resolve_single_file_path, + resolve_source, +) +from nf_docs.progress import ProgressCallbackType +from nf_docs.renderers import get_renderer + +__all__ = ["extract", "render", "generate"] + + +def _extract_resolved( + source: Path, + single_file_mode: bool, + *, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, +) -> Pipeline: + """ + Extract from an already-resolved source, so resolution happens exactly once. + + Args: + source: Resolved path, as returned by ``resolve_source`` + single_file_mode: Whether ``source`` is a single ``.nf`` file + + Returns: + The extracted Pipeline model + """ + extractor = PipelineExtractor( + workspace_path=find_pipeline_root(source) if single_file_mode else source, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + target_file=source if single_file_mode else None, + config=config, + ) + return extractor.extract() + + +def extract( + path: str | Path, + *, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, +) -> Pipeline: + """ + Extract documentation from a Nextflow pipeline or a single module file. + + ``path`` may be a pipeline directory or a single ``.nf`` file. A directory + containing only a module-style ``main.nf`` (process definitions, no + workflow, no pipeline config) is auto-detected as a single module — the same + rule the ``nf-docs generate`` command applies. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + config: Configuration to use. Defaults to :class:`NfDocsConfig` defaults. + Unlike the CLI, the user's ``~/.config/nf-docs/config.yaml`` is *not* + read automatically — pass ``config=load_config()`` to opt in. + language_server_jar: Path to the Nextflow Language Server JAR. Downloaded + on demand if not given. + nextflow_path: Path to the Nextflow executable + use_cache: Whether to read and write the extraction cache + force_refresh: Re-extract even when a cache entry exists + progress_callback: Called with :class:`~nf_docs.progress.ProgressUpdate` + objects as extraction proceeds + + Returns: + The extracted :class:`~nf_docs.models.Pipeline` model + + Raises: + ValueError: If ``path`` is a file that isn't a ``.nf`` file + ExtractionError: If extraction fails + LSPError: If the Language Server cannot be started or queried + """ + source, single_file_mode = resolve_source(path) + return _extract_resolved( + source, + single_file_mode, + config=config, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + ) + + +def render( + pipeline: Pipeline, + output_format: str = "html", + *, + title: str | None = None, + single_file: bool = False, + **renderer_kwargs: Any, +) -> str: + """ + Render a Pipeline model to a string. + + Args: + pipeline: The Pipeline model to render + output_format: One of ``html``, ``markdown`` (or ``md``), ``table``, + ``json``, ``yaml`` + title: Custom documentation title. Defaults to the pipeline name. + single_file: Render the focused single-document form used for modules + rather than the full pipeline form + **renderer_kwargs: Extra keyword arguments for the specific renderer, + e.g. ``use_tailwind`` (HTML), ``indent`` (JSON), + ``default_flow_style`` (YAML) + + Returns: + The rendered documentation as a string + + Raises: + ValueError: If ``output_format`` is not supported + """ + renderer = get_renderer(normalize_format(output_format))(title=title, **renderer_kwargs) + if single_file: + return renderer.render_single_file(pipeline) + return renderer.render(pipeline) + + +def generate( + path: str | Path, + *, + output_format: str = "html", + output: str | Path | None = None, + title: str | None = None, + config: NfDocsConfig | None = None, + language_server_jar: str | Path | None = None, + nextflow_path: str = "nextflow", + use_cache: bool = True, + force_refresh: bool = False, + progress_callback: ProgressCallbackType | None = None, + **renderer_kwargs: Any, +) -> list[Path]: + """ + Extract a pipeline and write its documentation to disk. + + This is the programmatic equivalent of ``nf-docs generate``. Unlike the CLI, + it always writes files rather than streaming to stdout — use :func:`extract` + plus :func:`render` if you want the documentation as a string. + + When ``output`` is not given, the defaults mirror the CLI: ``/docs`` + for a whole pipeline, and a file alongside the source (e.g. ``README.md``) + when documenting a single module. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + output_format: One of ``html``, ``markdown`` (or ``md``), ``table``, + ``json``, ``yaml`` + output: Output file or directory. Defaults to the CLI's conventions. + title: Custom documentation title + config, language_server_jar, nextflow_path, use_cache, force_refresh, + progress_callback: See :func:`extract`. + **renderer_kwargs: Extra keyword arguments for the specific renderer + + Returns: + The list of files written, in the order they were created + + Raises: + ValueError: If ``path`` or ``output_format`` is invalid + ExtractionError: If extraction fails + LSPError: If the Language Server cannot be started or queried + """ + canonical_format = normalize_format(output_format) + renderer = get_renderer(canonical_format)(title=title, **renderer_kwargs) + source, single_file_mode = resolve_source(path) + output_path = Path(output) if output is not None else None + + pipeline = _extract_resolved( + source, + single_file_mode, + config=config, + language_server_jar=language_server_jar, + nextflow_path=nextflow_path, + use_cache=use_cache, + force_refresh=force_refresh, + progress_callback=progress_callback, + ) + + if single_file_mode: + output_file = resolve_single_file_path(source, canonical_format, output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(renderer.render_single_file(pipeline), encoding="utf-8") + return [output_file] + + if canonical_format in DIRECTORY_FORMATS: + return renderer.render_to_directory( + pipeline, output_path if output_path is not None else default_output_dir(source) + ) + + output_file = resolve_data_file_path(source, canonical_format, output_path) + renderer.render_to_file(pipeline, output_file) + return [output_file] diff --git a/src/nf_docs/cache.py b/src/nf_docs/cache.py index 946f539..7b4b367 100644 --- a/src/nf_docs/cache.py +++ b/src/nf_docs/cache.py @@ -13,6 +13,7 @@ from typing import Any import nf_docs +from nf_docs.config import NfDocsConfig from nf_docs.models import ( ConfigParam, Function, @@ -133,17 +134,41 @@ def _get_target_file_hash(self, target_file: Path) -> str: logger.debug(f"Could not hash {meta}: {e}") return hasher.hexdigest()[:32] - def _get_cache_path(self, workspace: Path, target_file: Path | None = None) -> Path: - """Get the cache file path for a workspace (or a target file within it).""" + def _get_cache_path( + self, + workspace: Path, + target_file: Path | None = None, + config: NfDocsConfig | None = None, + ) -> Path: + """ + Get the cache file path for a workspace (or a target file within it). + + The configuration is folded into the content hash rather than added as a + separate filename component, which keeps ``_cleanup_old_caches`` working + unchanged. The trade-off is that alternating between two configs on one + pipeline evicts each entry in turn instead of keeping both; that costs a + re-extraction, where sharing an entry would return wrong results. + """ workspace_hash = self._get_workspace_hash(workspace) + config_hash = (config or NfDocsConfig()).cache_key() if target_file is not None: - target_hash = self._get_target_file_hash(target_file) + target_hash = self._combine(self._get_target_file_hash(target_file), config_hash) return self.cache_dir / f"{nf_docs.__version__}_{workspace_hash}_mod_{target_hash}.json" - content_hash = self._get_content_hash(workspace) + content_hash = self._combine(self._get_content_hash(workspace), config_hash) # Include version in filename to invalidate old caches when nf-docs is updated return self.cache_dir / f"{nf_docs.__version__}_{workspace_hash}_{content_hash}.json" - def get(self, workspace: Path, target_file: Path | None = None) -> Pipeline | None: + @staticmethod + def _combine(content_hash: str, config_hash: str) -> str: + """Fold a config hash into a content hash, keeping the digest length.""" + return hashlib.sha256(f"{content_hash}:{config_hash}".encode()).hexdigest() + + def get( + self, + workspace: Path, + target_file: Path | None = None, + config: NfDocsConfig | None = None, + ) -> Pipeline | None: """ Get cached Pipeline if valid. @@ -152,11 +177,14 @@ def get(self, workspace: Path, target_file: Path | None = None) -> Pipeline | No target_file: For single-file mode, the .nf file being extracted. The cache key becomes per-file so different modules within the same workspace don't collide. + config: Configuration the result was extracted with. Entries are + only reused for a matching config; defaults are assumed when + this is not given. Returns: Cached Pipeline if valid, None if cache miss or stale """ - cache_path = self._get_cache_path(workspace, target_file) + cache_path = self._get_cache_path(workspace, target_file, config) if not cache_path.exists(): logger.debug("Cache miss: no cache file") @@ -177,6 +205,7 @@ def set( workspace: Path, pipeline: Pipeline, target_file: Path | None = None, + config: NfDocsConfig | None = None, ) -> None: """ Store Pipeline in cache. @@ -185,10 +214,12 @@ def set( workspace: Path to the pipeline workspace pipeline: Pipeline model to cache target_file: For single-file mode, the .nf file being extracted. + config: Configuration the result was extracted with, so that a + different config doesn't read this entry back. """ self.cache_dir.mkdir(parents=True, exist_ok=True) - cache_path = self._get_cache_path(workspace, target_file) + cache_path = self._get_cache_path(workspace, target_file, config) # Clean up old cache files (per-workspace for pipeline mode, per-file # for single-file mode — we don't want one module to evict another). diff --git a/src/nf_docs/cli.py b/src/nf_docs/cli.py index 90d41c0..c5b947e 100644 --- a/src/nf_docs/cli.py +++ b/src/nf_docs/cli.py @@ -6,7 +6,6 @@ """ import logging -import re import sys from pathlib import Path @@ -23,8 +22,19 @@ ) from nf_docs import __version__ -from nf_docs.extractor import ExtractionError, PipelineExtractor, find_pipeline_root +from nf_docs.api import extract as extract_pipeline +from nf_docs.config import load_config +from nf_docs.extractor import ExtractionError, PipelineExtractor from nf_docs.lsp_client import LSPError +from nf_docs.output import ( + DIRECTORY_FORMATS, + default_output_dir, + normalize_format, + resolve_data_file_path, + resolve_single_file_path, + resolve_source, + streams_by_default, +) from nf_docs.progress import ProgressUpdate from nf_docs.renderers import get_renderer @@ -167,66 +177,6 @@ def _display_path(path: Path) -> str: return str(path) -def _looks_like_module(main_nf: Path) -> bool: - """ - Heuristic: ``main.nf`` defines at least one ``process`` block and no - top-level ``workflow`` block — i.e. it's an nf-core-style module rather - than a pipeline entry point or subworkflow. - """ - try: - content = main_nf.read_text(encoding="utf-8") - except OSError: - return False - has_process = re.search(r"(?m)^\s*process\b", content) is not None - has_workflow = re.search(r"(?m)^\s*workflow\b", content) is not None - return has_process and not has_workflow - - -# Single-file output policy per format: -# default_name: the filename to use when the user passes no --output (or -# passes a directory). ``None`` means "stream to stdout instead". -# dir_ext: when streaming-default formats are forced to a file via -# ``--output some_dir/``, use ``{source_stem}.{dir_ext}``. -_SINGLE_FILE_OUTPUT_POLICY: dict[str, tuple[str | None, str]] = { - "markdown": ("README.md", "md"), - "html": ("index.html", "html"), - "json": (None, "json"), - "yaml": (None, "yaml"), - "table": (None, "txt"), -} - - -def _resolve_single_file_output( - renderer, - pipeline, - source_file: Path, - output_format: str, - output_path: Path | None, -) -> tuple[Path | None, str]: - """ - For single-file mode, decide where to write and produce the rendered string. - - Returns a tuple of ``(output_file, rendered_content)``. ``output_file`` is - ``None`` when the content should go to stdout (json/yaml/table without - explicit ``--output``). - """ - rendered = renderer.render_single_file(pipeline) - default_name, dir_ext = _SINGLE_FILE_OUTPUT_POLICY.get(output_format, (None, output_format)) - - if output_path is not None: - # User-specified path always wins. Treat directories as "put a default - # filename in here" so `-o some_dir/` still works. - if output_path.is_dir(): - filename = default_name or f"{source_file.stem}.{dir_ext}" - return output_path / filename, rendered - return output_path, rendered - - if default_name is not None: - return source_file.parent / default_name, rendered - # json / yaml / table → stdout - return None, rendered - - @main.command() @click.argument("pipeline_path", type=click.Path(exists=True, path_type=Path)) @click.option( @@ -303,55 +253,33 @@ def generate( """ setup_logging(verbose) - # Normalize format aliases - if output_format == "md": - output_format = "markdown" + output_format = normalize_format(output_format) - # Decide single-file vs directory mode up front. - single_file_mode = pipeline_path.is_file() - if single_file_mode and pipeline_path.suffix.lower() != ".nf": - console.print(f"[red]Single-file input must be a .nf file, got: {pipeline_path.name}[/red]") + # Decide single-file vs directory mode up front. A directory holding only a + # module-style main.nf is auto-detected as a single module. + try: + pipeline_path, single_file_mode = resolve_source(pipeline_path) + except ValueError as e: + console.print(f"[red]{e}[/red]") sys.exit(1) - # Auto-detect a single module: directory has a main.nf that defines - # process(es) and no workflow block, with no pipeline-level config files. - if not single_file_mode: - main_nf = pipeline_path / "main.nf" - has_pipeline_config = (pipeline_path / "nextflow.config").is_file() or ( - pipeline_path / "nextflow_schema.json" - ).is_file() - if main_nf.is_file() and not has_pipeline_config and _looks_like_module(main_nf): - pipeline_path = main_nf - single_file_mode = True - if single_file_mode: logging.getLogger("nf_docs").info("Generating single module documentation") + # The CLI honours the user's ~/.config/nf-docs/config.yaml; the Python API + # deliberately does not (see nf_docs.api.extract). + user_config = load_config() + try: with ExtractionProgressDisplay(console) as progress_display: - # Extract documentation - if single_file_mode: - workspace_root = find_pipeline_root(pipeline_path) - extractor = PipelineExtractor( - workspace_path=workspace_root, - language_server_jar=language_server, - nextflow_path=nextflow_path, - use_cache=True, - force_refresh=no_cache, - progress_callback=progress_display.callback, - target_file=pipeline_path, - ) - else: - extractor = PipelineExtractor( - workspace_path=pipeline_path, - language_server_jar=language_server, - nextflow_path=nextflow_path, - use_cache=True, - force_refresh=no_cache, - progress_callback=progress_display.callback, - ) - - pipeline = extractor.extract() + pipeline = extract_pipeline( + pipeline_path, + config=user_config, + language_server_jar=language_server, + nextflow_path=nextflow_path, + force_refresh=no_cache, + progress_callback=progress_display.callback, + ) # Check if any content was found if not pipeline.has_content(): @@ -382,35 +310,31 @@ def generate( renderer = renderer_class(title=title) if single_file_mode: - output_file, rendered = _resolve_single_file_output( - renderer, pipeline, pipeline_path, output_format, output_path + rendered = renderer.render_single_file(pipeline) + # Formats with no natural single-file name go to stdout unless + # the user asked for a specific output path. + output_file = ( + None + if output_path is None and streams_by_default(output_format) + else resolve_single_file_path(pipeline_path, output_format, output_path) ) progress.update(task, description="Rendering complete") elif output_path: # Write to file/directory - if output_format in ("markdown", "html", "table"): + if output_format in DIRECTORY_FORMATS: created_files = renderer.render_to_directory(pipeline, output_path) - progress.update(task, description="Rendering complete") else: - # JSON/YAML - write to single file - # If output_path is a directory, use default filename - if output_path.is_dir(): - ext = "json" if output_format == "json" else "yaml" - output_file = output_path / f"pipeline.{ext}" - else: - output_file = output_path - output_file.parent.mkdir(parents=True, exist_ok=True) + # JSON/YAML - write to a single data file + output_file = resolve_data_file_path(pipeline_path, output_format, output_path) renderer.render_to_file(pipeline, output_file) - progress.update(task, description="Rendering complete") + progress.update(task, description="Rendering complete") else: # Write to stdout or default directory - if output_format in ("json", "yaml"): - pass # Will write to stdout after progress - else: - # Write to default directory - default_dir = pipeline_path / "docs" + if output_format in DIRECTORY_FORMATS: + default_dir = default_output_dir(pipeline_path) created_files = renderer.render_to_directory(pipeline, default_dir) progress.update(task, description="Rendering complete") + # json/yaml stream to stdout after the progress display is gone # Output results after progress display is gone if single_file_mode: @@ -425,7 +349,7 @@ def generate( ) console.print(f" - {_display_path(output_file)}") elif output_path: - if output_format in ("markdown", "html", "table"): + if output_format in DIRECTORY_FORMATS: file_word = "file" if len(created_files) == 1 else "files" console.print( f"[green]Created {len(created_files)} {file_word} in {output_path}[/green]" @@ -435,16 +359,16 @@ def generate( else: console.print(f"[green]Written to {output_file}[/green]") else: - if output_format in ("json", "yaml"): - # Write to stdout - click.echo(renderer.render(pipeline)) - else: + if output_format in DIRECTORY_FORMATS: file_word = "file" if len(created_files) == 1 else "files" console.print( f"[green]Created {len(created_files)} {file_word} in {default_dir}[/green]" ) for f in created_files: console.print(f" - ./{_display_path(f)}") + else: + # Write to stdout + click.echo(renderer.render(pipeline)) except LSPError as e: console.print(f"[red]Language Server error: {e}[/red]") @@ -483,6 +407,7 @@ def inspect(pipeline_path: Path, verbose: bool) -> None: extractor = PipelineExtractor( workspace_path=pipeline_path, progress_callback=progress_display.callback, + config=load_config(), ) pipeline = extractor.extract() diff --git a/src/nf_docs/config.py b/src/nf_docs/config.py index 3282ff7..af1ade0 100644 --- a/src/nf_docs/config.py +++ b/src/nf_docs/config.py @@ -8,6 +8,8 @@ if no config file exists. """ +import hashlib +import json import logging import os from dataclasses import dataclass, field @@ -116,6 +118,21 @@ def from_dict(cls, data: dict[str, Any]) -> "NfDocsConfig": exclude_patterns=data.get("exclude_patterns", DEFAULT_CONFIG["exclude_patterns"]), ) + def cache_key(self) -> str: + """ + Stable short hash of this configuration. + + Extraction results depend on the config in use, so the cache key has to + include it. Without this, a CLI run (which loads the user's config file) + and a library call (which uses defaults) would share a cache entry on + the same unchanged pipeline and return each other's results. + + Returns: + A hex digest suitable for use in a cache filename. + """ + payload = json.dumps(self.to_dict(), sort_keys=True) + return hashlib.sha256(payload.encode()).hexdigest()[:16] + def to_dict(self) -> dict[str, Any]: """Convert config to a dictionary.""" return { @@ -151,6 +168,15 @@ def load_config(config_path: Path | None = None) -> NfDocsConfig: with open(config_path, encoding="utf-8") as f: data = yaml.safe_load(f) or {} + if not isinstance(data, dict): + # Valid YAML, wrong shape (e.g. a top-level list). Treat it the same + # as unparseable rather than letting an AttributeError escape. + logger.warning( + f"Config file {config_path} must contain a mapping of settings, " + f"got {type(data).__name__}. Using defaults." + ) + return NfDocsConfig() + logger.debug(f"Loaded config from {config_path}") return NfDocsConfig.from_dict(data) @@ -217,28 +243,3 @@ def get_example_config() -> str: # - "tests/**/*.nf" # - "examples/**/*.nf" """ - - -# Global config instance (loaded lazily) -_config: NfDocsConfig | None = None - - -def get_config() -> NfDocsConfig: - """ - Get the global configuration instance. - - Loads the config from file on first call, then returns cached instance. - - Returns: - The global NfDocsConfig instance. - """ - global _config - if _config is None: - _config = load_config() - return _config - - -def reset_config() -> None: - """Reset the global config instance (useful for testing).""" - global _config - _config = None diff --git a/src/nf_docs/extractor.py b/src/nf_docs/extractor.py index 6caa444..d6bc1aa 100644 --- a/src/nf_docs/extractor.py +++ b/src/nf_docs/extractor.py @@ -17,7 +17,7 @@ from typing import Any from nf_docs.cache import PipelineCache -from nf_docs.config import get_config +from nf_docs.config import NfDocsConfig from nf_docs.config_parser import parse_config from nf_docs.git_utils import GitInfo, build_source_url, get_git_info from nf_docs.lsp_client import LSPClient, SymbolKind, parse_hover_content @@ -143,6 +143,7 @@ def __init__( force_refresh: bool = False, progress_callback: ProgressCallbackType | None = None, target_file: str | Path | None = None, + config: NfDocsConfig | None = None, ): """ Initialize the extractor. @@ -157,8 +158,12 @@ def __init__( target_file: If set, extract only from this single ``.nf`` file instead of scanning the whole workspace. Pipeline-level sources (schema, config, README, cache) are skipped in this mode. + config: Configuration to use. Defaults to ``NfDocsConfig()`` defaults + rather than the user's config file, so that library callers get + reproducible results. The CLI passes ``load_config()`` explicitly. """ self.workspace_path = Path(workspace_path).resolve() + self.config = config if config is not None else NfDocsConfig() self.language_server_jar = language_server_jar self.nextflow_path = nextflow_path self.target_file = Path(target_file).resolve() if target_file else None @@ -197,7 +202,9 @@ def extract(self) -> Pipeline: message="Checking cache...", ) ) - cached = self.cache.get(self.workspace_path, target_file=self.target_file) + cached = self.cache.get( + self.workspace_path, target_file=self.target_file, config=self.config + ) if cached: self._progress( ProgressUpdate( @@ -218,7 +225,12 @@ def extract(self) -> Pipeline: self._extract_from_lsp(pipeline, git_info) if self.cache: - self.cache.set(self.workspace_path, pipeline, target_file=self.target_file) + self.cache.set( + self.workspace_path, + pipeline, + target_file=self.target_file, + config=self.config, + ) self._progress( ProgressUpdate( @@ -264,11 +276,10 @@ def extract(self) -> Pipeline: pipeline.metadata = self._merge_metadata(pipeline.metadata, config_metadata) # Filter config params to exclude those already in inputs and ignored prefixes input_names = {inp.name for inp in pipeline.inputs} - config = get_config() pipeline.config_params = [ p for p in config_params - if p.name not in input_names and not config.should_ignore_config_param(p.name) + if p.name not in input_names and not self.config.should_ignore_config_param(p.name) ] except Exception as e: logger.warning(f"Failed to parse config: {e}") @@ -310,7 +321,7 @@ def extract(self) -> Pipeline: # Store in cache if self.cache: - self.cache.set(self.workspace_path, pipeline) + self.cache.set(self.workspace_path, pipeline, config=self.config) self._progress( ProgressUpdate( diff --git a/src/nf_docs/output.py b/src/nf_docs/output.py new file mode 100644 index 0000000..e5946b6 --- /dev/null +++ b/src/nf_docs/output.py @@ -0,0 +1,222 @@ +""" +Source and output resolution shared by the CLI and the Python API. + +This module answers two questions for both entry points: + +- what is being documented (a whole pipeline, or a single ``.nf`` module) +- where the rendered documentation goes on disk + +It holds no console output, no process exits, and no rendering. Keeping the +answers here is what stops ``nf_docs.cli`` and ``nf_docs.api`` from drifting +apart. It imports nothing from the rest of the package, so it stays cheap. +""" + +import re +from pathlib import Path + +# Format aliases accepted on input, mapped to their canonical renderer name. +# ``renderers.get_renderer()`` resolves through this too, so a new alias only +# needs adding here. +FORMAT_ALIASES: dict[str, str] = {"md": "markdown"} + +# Per-format single-file output policy: +# default_name: filename to use when no output path is given. ``None`` means +# the format has no natural single-file name, so the CLI streams it to +# stdout instead (see ``resolve_single_file_path``). +# extension: this format's file extension. +SINGLE_FILE_OUTPUT_POLICY: dict[str, tuple[str | None, str]] = { + "markdown": ("README.md", "md"), + "html": ("index.html", "html"), + "json": (None, "json"), + "yaml": (None, "yaml"), + "table": (None, "txt"), +} + +# Formats that render as a directory of files rather than a single document. +DIRECTORY_FORMATS: frozenset[str] = frozenset({"markdown", "html", "table"}) + +# Filename stem used for json/yaml output of a whole pipeline. +DATA_FILE_STEM = "pipeline" + +# Directory used when documenting a whole pipeline with no explicit output path. +DEFAULT_OUTPUT_DIR = "docs" + + +def normalize_format(output_format: str) -> str: + """ + Normalize a format name to its canonical renderer name. + + Args: + output_format: Format name, possibly an alias (e.g. ``md``) + + Returns: + The canonical format name (e.g. ``markdown``) + """ + lowered = output_format.lower() + return FORMAT_ALIASES.get(lowered, lowered) + + +def format_extension(output_format: str) -> str: + """ + Get the file extension for a canonical format name. + + Args: + output_format: Canonical format name + + Returns: + The extension, without a leading dot + + Raises: + KeyError: If the format has no output policy entry + """ + return SINGLE_FILE_OUTPUT_POLICY[output_format][1] + + +def looks_like_module(main_nf: Path) -> bool: + """ + Heuristic: ``main.nf`` defines at least one ``process`` block and no + top-level ``workflow`` block — i.e. it's an nf-core-style module rather + than a pipeline entry point or subworkflow. + + Args: + main_nf: Path to the ``main.nf`` file to inspect + + Returns: + ``True`` if the file looks like a standalone module + """ + try: + content = main_nf.read_text(encoding="utf-8") + except OSError: + return False + has_process = re.search(r"(?m)^\s*process\b", content) is not None + has_workflow = re.search(r"(?m)^\s*workflow\b", content) is not None + return has_process and not has_workflow + + +def resolve_source(path: str | Path) -> tuple[Path, bool]: + """ + Decide whether ``path`` should be documented as a single file or a pipeline. + + A ``.nf`` file is always single-file mode. A directory is auto-detected as a + single module when it holds a ``main.nf`` that defines process(es) and no + workflow block, and has no pipeline-level config files alongside it. + + Resolving an already-resolved path is a no-op, so callers that pass the + result back in get the same answer. + + Args: + path: Path to a Nextflow pipeline directory or a single ``.nf`` file + + Returns: + Tuple of ``(resolved_path, single_file_mode)``. In single-file mode the + resolved path points at the ``.nf`` file itself, not its directory. + + Raises: + ValueError: If ``path`` does not exist, or is a file that isn't a + ``.nf`` file + """ + source = Path(path) + + if not source.exists(): + # The CLI gets this from click's exists=True; library callers don't, and + # extracting a typo'd path otherwise yields an empty Pipeline named + # after the typo rather than an error. + raise ValueError(f"Path does not exist: {source}") + + if source.is_file(): + if source.suffix.lower() != ".nf": + raise ValueError(f"Single-file input must be a .nf file, got: {source.name}") + return source, True + + main_nf = source / "main.nf" + has_pipeline_config = (source / "nextflow.config").is_file() or ( + source / "nextflow_schema.json" + ).is_file() + if main_nf.is_file() and not has_pipeline_config and looks_like_module(main_nf): + return main_nf, True + + return source, False + + +def streams_by_default(output_format: str) -> bool: + """ + Whether a format has no natural single-file name, so the CLI streams it to + stdout when given no output path. + + This is a command-line concern: the Python API always writes a file. + + Args: + output_format: Canonical format name + + Returns: + ``True`` if the CLI streams this format rather than writing a file + + Raises: + KeyError: If the format has no output policy entry + """ + return SINGLE_FILE_OUTPUT_POLICY[output_format][0] is None + + +def resolve_single_file_path( + source_file: Path, + output_format: str, + output_path: Path | None, +) -> Path: + """ + Decide which file a single module's documentation is written to. + + Args: + source_file: The ``.nf`` file being documented + output_format: Canonical format name + output_path: Explicit output path, or ``None`` for the default + + Returns: + The file to write + + Raises: + KeyError: If the format has no output policy entry + """ + default_name, extension = SINGLE_FILE_OUTPUT_POLICY[output_format] + filename = default_name or f"{source_file.stem}.{extension}" + + if output_path is None: + return source_file.parent / filename + # User-specified path always wins. Treat directories as "put a default + # filename in here" so `-o some_dir/` still works. + if output_path.is_dir(): + return output_path / filename + return output_path + + +def resolve_data_file_path( + source: Path, + output_format: str, + output_path: Path | None, +) -> Path: + """ + Decide where a whole pipeline's json/yaml output goes. + + Args: + source: The pipeline directory being documented + output_format: Canonical format name (``json`` or ``yaml``) + output_path: Explicit output file or directory, or ``None`` for the + default of ``/docs/pipeline.`` + + Returns: + The file to write + + Raises: + KeyError: If the format has no output policy entry + """ + filename = f"{DATA_FILE_STEM}.{format_extension(output_format)}" + + if output_path is None: + return source / DEFAULT_OUTPUT_DIR / filename + if output_path.is_dir(): + return output_path / filename + return output_path + + +def default_output_dir(source: Path) -> Path: + """Get the default output directory for a whole pipeline.""" + return source / DEFAULT_OUTPUT_DIR diff --git a/src/nf_docs/py.typed b/src/nf_docs/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/nf_docs/renderers/__init__.py b/src/nf_docs/renderers/__init__.py index 1270db0..95a7216 100644 --- a/src/nf_docs/renderers/__init__.py +++ b/src/nf_docs/renderers/__init__.py @@ -9,6 +9,7 @@ Table: Compact Markdown tables (terraform-docs style) """ +from nf_docs.output import FORMAT_ALIASES, normalize_format from nf_docs.renderers.base import BaseRenderer from nf_docs.renderers.html import HTMLRenderer from nf_docs.renderers.json import JSONRenderer @@ -23,16 +24,28 @@ "YAMLRenderer", "MarkdownRenderer", "HTMLRenderer", + "RENDERERS", "get_renderer", ] +# Canonical format name -> renderer class. Aliases (e.g. "md") are resolved by +# ``nf_docs.output.normalize_format`` before lookup, so they live in one place. +RENDERERS: dict[str, type[BaseRenderer]] = { + "json": JSONRenderer, + "yaml": YAMLRenderer, + "markdown": MarkdownRenderer, + "html": HTMLRenderer, + "table": TableRenderer, +} + + def get_renderer(format: str) -> type[BaseRenderer]: """ Get the renderer class for a given format. Args: - format: Output format (json, yaml, markdown, html) + format: Output format (json, yaml, markdown/md, html, table) Returns: Renderer class @@ -40,18 +53,9 @@ def get_renderer(format: str) -> type[BaseRenderer]: Raises: ValueError: If format is not supported """ - renderers = { - "json": JSONRenderer, - "yaml": YAMLRenderer, - "markdown": MarkdownRenderer, - "md": MarkdownRenderer, - "html": HTMLRenderer, - "table": TableRenderer, - } - - format_lower = format.lower() - if format_lower not in renderers: - supported = ", ".join(sorted(set(renderers.keys()))) + canonical = normalize_format(format) + if canonical not in RENDERERS: + supported = ", ".join(sorted({*RENDERERS, *FORMAT_ALIASES})) raise ValueError(f"Unsupported format: {format}. Supported formats: {supported}") - return renderers[format_lower] + return RENDERERS[canonical] diff --git a/tests/conftest.py b/tests/conftest.py index 1b01333..de1afd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,20 @@ import pytest +@pytest.fixture(autouse=True) +def isolate_xdg_dirs(tmp_path_factory, monkeypatch) -> None: + """ + Point XDG cache and config at throwaway directories for every test. + + Without this, any test that extracts with caching left entries in the + developer's real ``~/.cache/nf-docs/`` - and because the Language Server is + mocked out in tests, those entries are empty results that a later real run + could pick up. + """ + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path_factory.mktemp("xdg_cache"))) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path_factory.mktemp("xdg_config"))) + + @pytest.fixture def sample_schema() -> dict: """Sample nextflow_schema.json content.""" diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..cf854ec --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,268 @@ +"""Tests for the high-level Python API (nf_docs.api).""" + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +import nf_docs +from nf_docs.api import extract, generate, render +from nf_docs.config import NfDocsConfig +from nf_docs.extractor import PipelineExtractor +from nf_docs.models import ConfigParam, Pipeline, PipelineMetadata, Process +from nf_docs.output import ( + looks_like_module, + normalize_format, + resolve_source, +) + + +@pytest.fixture +def module_dir(tmp_path: Path) -> Path: + """A directory holding a module-style main.nf (process, no workflow).""" + module = tmp_path / "modules" / "mytool" + module.mkdir(parents=True) + (module / "main.nf").write_text( + "process MYTOOL {\n input:\n path reads\n\n script:\n '''\n mytool\n '''\n}\n" + ) + return module + + +@pytest.fixture +def rendered_pipeline() -> Pipeline: + """A small Pipeline model that needs no extraction.""" + return Pipeline( + metadata=PipelineMetadata(name="Test Pipeline", description="A test"), + processes=[Process(name="FASTQC", docstring="Quality control")], + ) + + +class TestNormalizeFormat: + """Tests for format alias normalization.""" + + def test_md_alias(self) -> None: + assert normalize_format("md") == "markdown" + + def test_case_insensitive(self) -> None: + assert normalize_format("HTML") == "html" + + def test_passthrough(self) -> None: + assert normalize_format("json") == "json" + + +class TestResolveSource: + """Tests for single-file vs pipeline source resolution.""" + + def test_pipeline_directory(self, sample_pipeline: Path) -> None: + resolved, single_file = resolve_source(sample_pipeline) + assert resolved == sample_pipeline + assert single_file is False + + def test_nf_file(self, sample_pipeline: Path) -> None: + resolved, single_file = resolve_source(sample_pipeline / "main.nf") + assert resolved == sample_pipeline / "main.nf" + assert single_file is True + + def test_module_directory_autodetected(self, module_dir: Path) -> None: + """A bare module dir resolves to its main.nf in single-file mode.""" + resolved, single_file = resolve_source(module_dir) + assert resolved == module_dir / "main.nf" + assert single_file is True + + def test_non_nf_file_rejected(self, tmp_path: Path) -> None: + other = tmp_path / "notes.txt" + other.write_text("hello") + with pytest.raises(ValueError, match="must be a .nf file"): + resolve_source(other) + + def test_looks_like_module(self, module_dir: Path, sample_pipeline: Path) -> None: + assert looks_like_module(module_dir / "main.nf") is True + # The sample pipeline's main.nf defines a workflow, so it isn't a module + assert looks_like_module(sample_pipeline / "main.nf") is False + + +class TestExtract: + """Tests for nf_docs.extract().""" + + def test_extract_directory(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + pipeline = extract(sample_pipeline) + assert isinstance(pipeline, Pipeline) + assert pipeline.inputs + + def test_extract_single_file(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + pipeline = extract(sample_pipeline / "main.nf") + assert isinstance(pipeline, Pipeline) + + def test_rejects_non_nf_file(self, tmp_path: Path) -> None: + other = tmp_path / "notes.txt" + other.write_text("hello") + with pytest.raises(ValueError): + extract(other) + + def test_rejects_missing_path(self, tmp_path: Path) -> None: + """A typo'd path must raise, not return an empty Pipeline named after it.""" + with pytest.raises(ValueError, match="does not exist"): + extract(tmp_path / "no_such_pipeline") + + def test_cache_is_keyed_on_config(self, sample_pipeline: Path) -> None: + """ + Two configs must not share a cache entry. + + Without the config in the cache key a CLI run (user config) and a + library call (defaults) return each other's results for an unchanged + pipeline. + """ + params = (PipelineMetadata(), [ConfigParam(name="genomes.mm10")]) + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch("nf_docs.extractor.parse_config", return_value=params): + # Caching left ON deliberately - that is the thing under test. + permissive = extract( + sample_pipeline, config=NfDocsConfig(ignore_config_prefixes=[]) + ) + restrictive = extract( + sample_pipeline, config=NfDocsConfig(ignore_config_prefixes=["genomes."]) + ) + assert [p.name for p in permissive.config_params] == ["genomes.mm10"] + assert [p.name for p in restrictive.config_params] == [] + + def test_config_is_injected(self, sample_pipeline: Path) -> None: + """A passed config reaches the extractor rather than the global one.""" + config = NfDocsConfig(ignore_config_prefixes=["params."]) + with patch("nf_docs.api.PipelineExtractor") as mock_cls: + extract(sample_pipeline, config=config) + assert mock_cls.call_args.kwargs["config"] is config + + def test_defaults_are_hermetic(self, sample_pipeline: Path) -> None: + """Without a config argument, defaults are used - not the user's file.""" + assert PipelineExtractor(workspace_path=sample_pipeline).config == NfDocsConfig() + + def test_config_filters_config_params(self, sample_pipeline: Path) -> None: + """The injected config actually affects the extracted result.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + with patch( + "nf_docs.extractor.parse_config", + return_value=(PipelineMetadata(), [ConfigParam(name="genomes.hg38")]), + ): + default = extract(sample_pipeline, use_cache=False) + permissive = extract( + sample_pipeline, + use_cache=False, + config=NfDocsConfig(ignore_config_prefixes=[]), + ) + assert [p.name for p in default.config_params] == [] + assert [p.name for p in permissive.config_params] == ["genomes.hg38"] + + def test_progress_callback_is_called(self, sample_pipeline: Path) -> None: + updates = [] + with patch.object(PipelineExtractor, "_extract_from_lsp"): + extract(sample_pipeline, use_cache=False, progress_callback=updates.append) + assert updates + assert all(hasattr(u, "phase") for u in updates) + + +class TestRender: + """Tests for nf_docs.render().""" + + @pytest.mark.parametrize("output_format", ["json", "yaml", "markdown", "html", "table"]) + def test_all_formats(self, rendered_pipeline: Pipeline, output_format: str) -> None: + result = render(rendered_pipeline, output_format) + assert isinstance(result, str) + assert result + + def test_md_alias(self, rendered_pipeline: Pipeline) -> None: + assert render(rendered_pipeline, "md") == render(rendered_pipeline, "markdown") + + def test_json_is_parseable(self, rendered_pipeline: Pipeline) -> None: + data = json.loads(render(rendered_pipeline, "json")) + assert data["pipeline"]["name"] == "Test Pipeline" + + def test_title_override(self, rendered_pipeline: Pipeline) -> None: + assert "Custom Title" in render(rendered_pipeline, "markdown", title="Custom Title") + + def test_renderer_kwargs_are_passed(self, rendered_pipeline: Pipeline) -> None: + compact = render(rendered_pipeline, "json", indent=0) + spaced = render(rendered_pipeline, "json", indent=8) + assert len(spaced) > len(compact) + + def test_single_file_mode(self, rendered_pipeline: Pipeline) -> None: + result = render(rendered_pipeline, "markdown", single_file=True) + assert isinstance(result, str) + assert result + + def test_unsupported_format(self, rendered_pipeline: Pipeline) -> None: + with pytest.raises(ValueError, match="Unsupported format"): + render(rendered_pipeline, "pdf") + + +class TestGenerate: + """Tests for nf_docs.generate().""" + + def test_returns_written_paths(self, sample_pipeline: Path, tmp_path: Path) -> None: + out = tmp_path / "site" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="html", output=out) + assert created + assert all(p.exists() for p in created) + + def test_json_to_explicit_file(self, sample_pipeline: Path, tmp_path: Path) -> None: + target = tmp_path / "api.json" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="json", output=target) + assert created == [target] + assert json.loads(target.read_text()) + + def test_json_default_location(self, sample_pipeline: Path) -> None: + """Without an output, json lands in /docs/pipeline.json.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="json") + assert created == [sample_pipeline / "docs" / "pipeline.json"] + assert created[0].exists() + + def test_markdown_default_directory(self, sample_pipeline: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(sample_pipeline, output_format="markdown") + assert created + assert all(p.is_relative_to(sample_pipeline / "docs") for p in created) + + def test_single_file_writes_readme(self, module_dir: Path) -> None: + """A module gets a README.md next to its main.nf.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(module_dir, output_format="md") + assert created == [module_dir / "README.md"] + assert created[0].exists() + + def test_single_file_json_writes_a_file(self, module_dir: Path) -> None: + """Unlike the CLI, generate() writes json to disk rather than stdout.""" + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate(module_dir, output_format="json") + assert created == [module_dir / "main.json"] + assert created[0].exists() + + def test_title_reaches_output(self, sample_pipeline: Path, tmp_path: Path) -> None: + with patch.object(PipelineExtractor, "_extract_from_lsp"): + created = generate( + sample_pipeline, + output_format="markdown", + output=tmp_path / "docs", + title="My Custom Pipeline", + ) + assert any("My Custom Pipeline" in p.read_text() for p in created) + + +class TestPublicSurface: + """The names promised in nf_docs.__all__ must resolve.""" + + def test_all_names_resolve(self) -> None: + missing = [name for name in nf_docs.__all__ if not hasattr(nf_docs, name)] + assert missing == [] + + def test_facade_is_exported(self) -> None: + assert nf_docs.extract is extract + assert nf_docs.render is render + assert nf_docs.generate is generate + + def test_py_typed_marker_ships(self) -> None: + assert (Path(nf_docs.__file__).parent / "py.typed").exists() diff --git a/tests/test_config.py b/tests/test_config.py index 8c6fcdc..126ac95 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,7 +11,6 @@ get_example_config, get_xdg_config_home, load_config, - reset_config, ) @@ -147,13 +146,6 @@ def test_to_dict(self) -> None: class TestLoadConfig: """Tests for load_config function.""" - @pytest.fixture(autouse=True) - def reset(self) -> None: - """Reset global config before and after each test.""" - reset_config() - yield - reset_config() - def test_load_default_when_file_missing(self, tmp_path: Path) -> None: """Test that defaults are used when config file doesn't exist.""" config = load_config(tmp_path / "nonexistent" / "config.yaml") @@ -161,6 +153,20 @@ def test_load_default_when_file_missing(self, tmp_path: Path) -> None: assert config.ignore_config_prefixes == ["genomes."] assert config.include_hidden_params is False + @pytest.mark.parametrize( + "contents", + ["- ignore_config_prefixes\n- genomes.\n", "just a string\n", "42\n"], + ids=["list", "string", "number"], + ) + def test_load_default_when_yaml_is_not_a_mapping(self, tmp_path: Path, contents: str) -> None: + """Valid YAML of the wrong shape falls back to defaults, it doesn't raise.""" + config_file = tmp_path / "config.yaml" + config_file.write_text(contents) + + config = load_config(config_file) + + assert config == NfDocsConfig() + def test_load_from_file(self, tmp_path: Path) -> None: """Test loading config from a valid YAML file.""" config_file = tmp_path / "config.yaml" diff --git a/zensical.toml b/zensical.toml index 93df0d6..35ef531 100644 --- a/zensical.toml +++ b/zensical.toml @@ -11,6 +11,7 @@ nav = [ {"Installation" = "installation.md"}, {"Running nf-docs" = "running.md"}, {"Writing docs" = "writing.md"}, + {"Python API" = "python-api.md"}, {"Examples" = "examples.md"}, ]