Replace update_references_file with nf-schema param substitution

CHANGELOG.md

@@ -65,16 +65,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#157](https://github.com/nf-core/references/pull/157) - Replace `.out` channel access with typed workflow return assignment
 - [#161](https://github.com/nf-core/references/pull/161) - Document `references_base_path` description and igenomes prefix-rewriting behaviour
 - [#161](https://github.com/nf-core/references/pull/161) - Replace placeholder `TODO` error messages in `schema_references.json`
+- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow
 - [#162](https://github.com/nf-core/references/pull/162) - Resolve datasheet `splice_sites` paths to files so publishing works on Nextflow 26.10
 - [#163](https://github.com/nf-core/references/pull/163) - Update nf-schema to 3.0.0 and migrate `paramsHelp`, `paramsSummaryLog`, and `paramsSummaryMap` to their single-options-map signatures
-- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow
+- [#164](https://github.com/nf-core/references/pull/164) - Replace `update_references_file` string rewriting with nf-schema `allowParamsSubstitution`, so datasheets can carry `${params.references_base_path}` placeholders
 
 ### Removed
 
-- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome`
-
 - [#146](https://github.com/nf-core/references/pull/146) - Removed `index.json` output in favor of Nextflow lineage tracking
 
+- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome`
+
 ### Fixed
 
 - [#77](https://github.com/nf-core/references/pull/77) - Fix createsequencedictionary usage

docs/usage.md

@@ -158,7 +158,28 @@ TODO
 
 TODO
 
-An [example asset](../assets/asset.yml) has been provided with the pipeline.
+Example asset files are published in the [nf-core/references-datasheets](https://github.com/nf-core/references-datasheets) repository.
+
+### `--references_base_path`
+
+A datasheet path may carry a `${params.references_base_path}` placeholder instead of a hardcoded root:
+
+```yaml
+- genome: GRCh38_chr21
+  fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa
+```
+
+The pipeline substitutes the parameter into the path as the datasheet loads, so one datasheet can be used against any mirror of the reference files:
+
+```bash
+--references_base_path '/data/mirror/igenomes'
+```
+
+The default is `s3://ngi-igenomes/igenomes`, the root that published datasheets are written against. With that default the example above resolves to `s3://ngi-igenomes/igenomes/GRCh38_chr21/GRCh38_chr21.fa`.
+
+The placeholder carries the leading slash, so the value must not end with one. A trailing slash produces a doubled slash in the path.
+
+Datasheets without placeholders are unchanged and need no edit. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with `--input`.
 
 ## Running the pipeline
 

main.nf

@@ -295,8 +295,6 @@ workflow {
         params.help,
         params.help_full,
         params.show_hidden,
-        params.references_base_path,
-        ['s3://ngi-igenomes/igenomes/'],
         tools,
     )
 

nextflow.config

@@ -12,6 +12,7 @@ params {
     monochrome_logs              = false
     pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/'
     trace_report_suffix          = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss')
+    references_base_path         = 's3://ngi-igenomes/igenomes'
 
     custom_config_version        = 'master'
     custom_config_base           = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
@@ -248,7 +249,7 @@ manifest {
 
 // Nextflow plugins
 plugins {
-    id 'nf-core-utils@0.4.0' // nf-core-utils is a collection of utilities for Nextflow pipelines
+    id 'nf-core-utils@0.5.0' // nf-core-utils is a collection of utilities for Nextflow pipelines
     id 'nf-schema@3.0.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet
 }
 
@@ -257,6 +258,8 @@ lineage.enabled = true
 
 validation {
     monochromeLogs = params.monochrome_logs
+    allowParamsSubstitution = true
 }
+
 // Load modules.config for DSL2 module specific options
 includeConfig 'conf/modules.config'

nextflow_schema.json

@@ -16,7 +16,6 @@
                     "type": "string",
                     "format": "file-path",
                     "exists": true,
-                    "schema": "subworkflows/local/utils_references/schema_references.json",
                     "mimetype": "text/yaml",
                     "pattern": "^\\S+\\.(yaml|yml)$",
                     "errorMessage": "Input datasheet must be an existing file ending in '.yaml' or '.yml' that does not contain spaces",
@@ -57,8 +56,8 @@
                 "references_base_path": {
                     "type": "string",
                     "format": "directory-path",
-                    "description": "Base path used to resolve reference file paths in the datasheet, replacing the default 's3://ngi-igenomes/igenomes/' prefix.",
-                    "help_text": "Datasheets may carry igenomes-style 's3://ngi-igenomes/igenomes/...' paths. When this parameter is set, that prefix is rewritten to this base path before files are resolved, so you can point at a local or institutional mirror of igenomes without editing the datasheet.",
+                    "description": "Base path substituted for ${params.references_base_path} placeholders in datasheet paths.",
+                    "help_text": "Datasheet file paths may contain a ${params.references_base_path} placeholder instead of a hardcoded root, for example ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa. The placeholder is replaced by this parameter when the datasheet loads, so one datasheet can be used against any mirror of the reference files. The default is s3://ngi-igenomes/igenomes, the root that published datasheets are written against. The placeholder carries the leading slash, so the value must not end with one. Datasheets with no placeholder are unaffected. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with --input.",
                     "fa_icon": "fas fa-ban"
                 },
                 "email": {

subworkflows/local/utils_nfcore_references_pipeline/main.nf

@@ -8,8 +8,6 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */
 
-include { update_references_file } from '../../local/utils_references'
-
 include { checkCondaChannels     } from 'plugin/nf-core-utils'
 include { checkConfigProvided    } from 'plugin/nf-core-utils'
 include { checkProfileProvided   } from 'plugin/nf-core-utils'
@@ -38,8 +36,6 @@ workflow PIPELINE_INITIALISATION {
     nextflow_cli_args //   array: List of positional nextflow CLI args
     outdir //  string: The output directory where the results will be saved
     asset //  string: Path to asset yaml file
-    basepath_final //  string: The final basepath to replace in the asset yaml file
-    basepath_to_replace //  array: The basepath to replace in the asset yaml file
     help // boolean: Display help message and exit
     help_full // boolean: Show the full help message
     show_hidden // boolean: Show hidden parameters in the help message
@@ -158,7 +154,7 @@ workflow PIPELINE_INITIALISATION {
     // Create channel from asset file provided through params.asset
     references = channel.fromList(
         samplesheetToList(
-            update_references_file(asset, basepath_final, basepath_to_replace),
+            asset,
             "${projectDir}/subworkflows/local/utils_references/schema_references.json",
         )
     )

subworkflows/local/utils_references/main.nf

@@ -12,14 +12,12 @@ workflow UTILS_REFERENCES {
     param_value
     attribute_file
     attribute_value
-    basepath_final
-    basepath_to_replace
 
     main:
 
     references = channel.fromList(
         samplesheetToList(
-            update_references_file(yaml_reference, basepath_final, basepath_to_replace),
+            yaml_reference,
             "${projectDir}/subworkflows/local/utils_references/schema_references.json",
         )
     )
@@ -40,26 +38,6 @@ workflow UTILS_REFERENCES {
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */
 
-// Change all igenomes base parameter values in the file to the correct value
-def update_references_file(yaml_reference, basepath_final, basepath_to_replace) {
-    def correct_yaml_file = file(yaml_reference, checkIfExists: true)
-
-    if (basepath_final) {
-        def staged_yaml_file = "${workflow.workDir}/tmp/${java.util.UUID.randomUUID().toString()}.${correct_yaml_file.extension}"
-        correct_yaml_file.copyTo(staged_yaml_file)
-        correct_yaml_file = file(staged_yaml_file, checkIfExists: true)
-
-        // Use a local variable to accumulate changes
-        def updated_yaml_content = correct_yaml_file.text
-        basepath_to_replace.each { basepath_replacement ->
-            updated_yaml_content = updated_yaml_content.replace(basepath_replacement, basepath_final)
-        }
-        correct_yaml_file.text = updated_yaml_content
-    }
-
-    return correct_yaml_file
-}
-
 def get_references_file(references, param, attribute) {
     return references
         .map { meta, _readme ->

tests/assets/GRCh38_chr21_placeholder.yml

@@ -0,0 +1,8 @@
+# Test fixture for ${params.references_base_path} substitution (nf-schema
+# validation.allowParamsSubstitution). The placeholder carries the leading
+# slash; the base path must not end with one.
+- genome: GRCh38_chr21
+  fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa
+  mito_name: MT
+  source: nf-core
+  species: Homo_sapiens

tests/basepath_substitution.nf.test

@@ -0,0 +1,42 @@
+nextflow_pipeline {
+
+    name "Reference base path substitution"
+    script "../main.nf"
+    tag "pipeline"
+
+    test('--tools faidx with a datasheet carrying placeholders') {
+
+        options "-output-dir $outputDir"
+
+        when {
+            params {
+                input                = "${projectDir}/tests/assets/GRCh38_chr21_placeholder.yml"
+                outdir               = "$outputDir"
+                tools                = 'faidx'
+                tools_bundle         = null
+                skip_tools           = null
+                references_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/references/references'
+            }
+        }
+
+        then {
+            // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name)
+            def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
+            // stable_content: All files in ${params.outdir}/ with stable content
+            def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            assert workflow.success
+            assertAll(
+                { assert snapshot(
+                    // Number of successful tasks
+                    workflow.trace.succeeded().size(),
+                    // pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key)
+                    removeFromYamlMap("${outputDir}/pipeline_info/nf_core_references_software_mqc_versions.yml", 'Workflow'),
+                    // All stable path name, with a relative path
+                    stable_path,
+                    // All files with stable contents
+                    stable_content
+                ).match() }
+            )
+        }
+    }
+}

tests/basepath_substitution.nf.test.snap

@@ -0,0 +1,41 @@
+{
+    "--tools faidx with a datasheet carrying placeholders": {
+        "content": [
+            5,
+            {
+                "SAMTOOLS_FAIDX": {
+                    "samtools": "1.24"
+                }
+            },
+            [
+                "Homo_sapiens",
+                "Homo_sapiens/nf-core",
+                "Homo_sapiens/nf-core/GRCh38_chr21",
+                "Homo_sapiens/nf-core/GRCh38_chr21/Sequence",
+                "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta",
+                "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta/GRCh38_chr21.fa.fai",
+                "multiqc",
+                "multiqc/multiqc_data",
+                "multiqc/multiqc_data/llms-full.txt",
+                "multiqc/multiqc_data/multiqc.log",
+                "multiqc/multiqc_data/multiqc.parquet",
+                "multiqc/multiqc_data/multiqc_citations.txt",
+                "multiqc/multiqc_data/multiqc_data.json",
+                "multiqc/multiqc_data/multiqc_software_versions.txt",
+                "multiqc/multiqc_data/multiqc_sources.txt",
+                "multiqc/multiqc_report.html",
+                "pipeline_info",
+                "pipeline_info/nf_core_references_software_mqc_versions.yml"
+            ],
+            [
+                "GRCh38_chr21.fa.fai:md5,6ea51e1c1231da5e49212ea9f493d703",
+                "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f"
+            ]
+        ],
+        "timestamp": "2026-10-23T09:24:07.623957777",
+        "meta": {
+            "nf-test": "0.9.5",
+            "nextflow": "26.10.0"
+        }
+    }
+}