Add image QC and transcript QC subworkflow (ported from nf-xenium-processing dev HEAD)

.github/workflows/nf-test.yml

@@ -64,7 +64,10 @@ jobs:
     runs-on: # use self-hosted runners
       - runs-on=${{ github.run_id }}-nf-test
       - runner=4cpu-linux-x64
-      - volume=80gb
+      # 80 GB is no longer enough: a shard that pulls xeniumranger (which bundles
+      # cellranger) alongside the image QC and transcript QC containers ran out of
+      # disk with `docker: failed to register layer: no space left on device`.
+      - volume=120gb
     strategy:
       fail-fast: false
       matrix:

.github/workflows/python-tests.yml

@@ -0,0 +1,56 @@
+name: Python tests
+# Runs the Python unit tests for the QC analysis scripts.
+#
+# The nf-test suites exercise the Nextflow layer, and the pipeline-level QC
+# tests run with `-stub`, so without this workflow none of the QC Python code is
+# ever executed in CI.
+#
+# The scripts import their heavy dependencies at module scope, so the tests need
+# a real runtime environment (matplotlib, tifffile, scanpy, ...), not a linting
+# environment. The env is therefore built from the QC modules' own
+# environment.yml files plus the test-only extras, which keeps every runtime pin
+# declared in exactly one place.
+on:
+  push:
+    branches:
+      - dev
+      - master
+  pull_request:
+  release:
+    types: [published]
+
+env:
+  NXF_ANSI_LOG: false
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  pytest:
+    name: Run Python tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Set up the QC runtime environment
+        uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2
+        with:
+          environment-name: qc-tests
+          # One self-contained environment, not a merge of the two QC module
+          # environments: those cannot be co-installed (image QC pins
+          # pyarrow 21, while transcript QC's anndata requires pyarrow <21), so
+          # the test environment declares its own coherent set.
+          environment-file: bin/tests/environment.yml
+          cache-environment: true
+
+      - name: Show environment
+        shell: bash -el {0}
+        run: |
+          python --version
+          python -c "import numpy, pandas, matplotlib, tifffile, scanpy; print('runtime imports OK')"
+
+      - name: Run pytest
+        shell: bash -el {0}
+        run: pytest tests/ bin/tests/ -v -n auto

.gitignore

@@ -10,3 +10,5 @@ null/
 .nf-test/
 .nf-test.log
 .nf-test-*
+.mypy_cache/
+.vscode/

.nf-core.yml

@@ -10,6 +10,11 @@ lint:
     - docs/images/nf-core-spatialaxe_logo_dark.png
     - docs/images/nf-core-spatialaxe_logo_light.png
     - .github/PULL_REQUEST_TEMPLATE.md
+  # The QC report notebooks use doubled braces as Python f-string escapes (they
+  # emit pandoc callout-note divs), which this check reads as leftover Jinja.
+  template_strings:
+    - bin/xenium_image_qc_report.qmd
+    - bin/transcript_qc.qmd
 nf_core_version: 4.0.3
 repository_type: pipeline
 template:

CHANGELOG.md

@@ -19,6 +19,10 @@ Initial release of nf-core/spatialaxe, created with the [nf-core](https://nf-co.
 - samplesheet redefinition: `sample,bundle,image,annotation,stainings`, samplesheet allows for two additional optional columns `annotation,stainings` that are useful for the QC subworkflow.
 - `spatialdata_write_meta_merge/main.nf`: Change to subworkflow to account for proper `qc` mode.
 - Change to `bin/spatialdata_write.py`: Adding an `all` mode to set all available features to `True`, which is important for QC.
+- **Image QC and transcript QC**, ported from the internal nf-xenium-processing repository. Adds the `QC` subworkflow (`subworkflows/local/qc/`) wrapping `IMAGE_QC` and `TRANSCRIPT_QC` (`subworkflows/local/image_qc/`, `subworkflows/local/transcript_qc/`), the `image_qc` and `transcript_qc` local modules, and their analysis scripts, report notebooks and threshold configs in `bin/`. Image QC computes focus, SNR and morphology metrics; transcript QC computes per-transcript, per-cell and per-FoV metrics with a bounded-memory streaming reader. Each renders an HTML report with the nf-core `quarto/notebook` module.
+- 17 new QC parameters under the `qc_options` schema group, including `image_qc_gpus` (image QC is GPU-optional; the value is both the accelerator request and the device cap passed to the script), `tile_size`, `neg_control_prefix`, and figure/SNR/streaming toggles.
+- `conf/base.config`: new `process_gpu_qc` label for the optionally-GPU image QC step, routed to the GPU queue in the `aws` profile.
+- `.github/workflows/python-tests.yml` plus `bin/tests/`: the QC analysis scripts' Python unit tests, which were previously never executed in CI. Run in a runtime environment (`bin/tests/environment.yml`) because the scripts import their heavy dependencies at module scope.
 
 ### `Fixed`
 

bin/baysor_create_dataset.py

@@ -13,18 +13,19 @@
 from pathlib import Path
 
 
-class BaysorPreview():
+class BaysorPreview:
     """
     Utility class to generate baysor preview dataset
     """
+
     @staticmethod
     def generate_dataset(
-            transcripts: Path,
-            sampled_transcripts: Path,
-            sample_fraction: float = 0.3,
-            random_state: int = 42,
-            prefix: str = ""
-        ) -> None:
+        transcripts: Path,
+        sampled_transcripts: Path,
+        sample_fraction: float = 0.3,
+        random_state: int = 42,
+        prefix: str = "",
+    ) -> None:
         """
         Reads a csv file & randomly samples a fraction of rows,
         and writes the result to a .csv file.
@@ -40,9 +41,10 @@ def generate_dataset(
         random.seed(random_state)
         output_path = f"{prefix}/{sampled_transcripts}"
         os.makedirs(os.path.dirname(output_path), exist_ok=True)
-        with open(transcripts, mode='rt', newline='') as infile, \
-            open(output_path, mode='wt', newline='') as outfile:
-
+        with (
+            open(transcripts, mode="rt", newline="") as infile,
+            open(output_path, mode="wt", newline="") as outfile,
+        ):
             reader = csv.reader(infile)
             writer = csv.writer(outfile)
 
@@ -66,27 +68,25 @@ def main() -> None:
         description="Create sampled dataset for Baysor preview"
     )
     parser.add_argument(
-        "--transcripts", required=True,
-        help="Path to transcripts CSV file"
-    )
-    parser.add_argument(
-        "--sample-fraction", required=True, type=float,
-        help="Fraction of rows to sample"
+        "--transcripts", required=True, help="Path to transcripts CSV file"
     )
     parser.add_argument(
-        "--prefix", required=True,
-        help="Output directory prefix"
+        "--sample-fraction",
+        required=True,
+        type=float,
+        help="Fraction of rows to sample",
     )
+    parser.add_argument("--prefix", required=True, help="Output directory prefix")
     args = parser.parse_args()
 
-    sampled_transcripts = "sampled_transcripts.csv"
+    sampled_transcripts = Path("sampled_transcripts.csv")
 
     # generate dataset
     BaysorPreview.generate_dataset(
-        transcripts=args.transcripts,
+        transcripts=Path(args.transcripts),
         sampled_transcripts=sampled_transcripts,
         sample_fraction=args.sample_fraction,
-        prefix=args.prefix
+        prefix=args.prefix,
     )
 
     return None