From e44f08a262f30da6d29030e725653fc3bd35b6e3 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 22 Oct 2026 11:06:08 +0200 Subject: [PATCH 1/4] Replace update_references_file with nf-schema param substitution Drop the whole-YAML string rewrite that staged a copy into workflow.workDir/tmp and rewrote a hardcoded igenomes prefix. nf-schema 3.0.0 substitutes ${params.X} placeholders in datasheet values at load time, so datasheets can carry a base path instead of a hardcoded root. Drop the 'schema' key from the input param: it deep-validated the datasheet during validateParameters(), before substitution could run. samplesheetToList validates against the same schema afterwards. Generated by OpenCode --- CHANGELOG.md | 7 ++-- main.nf | 2 -- nextflow.config | 6 ++++ nextflow_schema.json | 1 - .../utils_nfcore_references_pipeline/main.nf | 6 +--- subworkflows/local/utils_references/main.nf | 24 +------------ tests/assets/GRCh38_chr21_placeholder.yml | 8 +++++ tests/basepath_substitution.nf.test | 34 +++++++++++++++++++ 8 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 tests/assets/GRCh38_chr21_placeholder.yml create mode 100644 tests/basepath_substitution.nf.test diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cf6c392..479054cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,16 +65,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#157](https://github.com/nf-core/references/pull/157) - Replace `.out` channel access with typed workflow return assignment - [#161](https://github.com/nf-core/references/pull/161) - Document `references_base_path` description and igenomes prefix-rewriting behaviour - [#161](https://github.com/nf-core/references/pull/161) - Replace placeholder `TODO` error messages in `schema_references.json` +- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow - [#162](https://github.com/nf-core/references/pull/162) - Resolve datasheet `splice_sites` paths to files so publishing works on Nextflow 26.10 - [#163](https://github.com/nf-core/references/pull/163) - Update nf-schema to 3.0.0 and migrate `paramsHelp`, `paramsSummaryLog`, and `paramsSummaryMap` to their single-options-map signatures -- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow +- [#164](https://github.com/nf-core/references/pull/164) - Replace `update_references_file` string rewriting with nf-schema `allowParamsSubstitution`, so datasheets can carry `${params.references_base_path}` placeholders ### Removed -- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome` - - [#146](https://github.com/nf-core/references/pull/146) - Removed `index.json` output in favor of Nextflow lineage tracking +- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome` + ### Fixed - [#77](https://github.com/nf-core/references/pull/77) - Fix createsequencedictionary usage diff --git a/main.nf b/main.nf index 488b3968..a98c8d28 100644 --- a/main.nf +++ b/main.nf @@ -295,8 +295,6 @@ workflow { params.help, params.help_full, params.show_hidden, - params.references_base_path, - ['s3://ngi-igenomes/igenomes/'], tools, ) diff --git a/nextflow.config b/nextflow.config index 065725c5..d8284313 100644 --- a/nextflow.config +++ b/nextflow.config @@ -12,6 +12,9 @@ params { monochrome_logs = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' trace_report_suffix = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') + // Default root substituted for ${params.references_base_path} placeholders in datasheets. + // Unconditional so substitution can never leave a literal placeholder or a null prefix in a path. + references_base_path = 's3://ngi-igenomes/igenomes/' custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -257,6 +260,9 @@ lineage.enabled = true validation { monochromeLogs = params.monochrome_logs + // Substitute ${params.X} placeholders in datasheet values at load time, replacing + // the previous update_references_file whole-YAML rewrite. + allowParamsSubstitution = true } // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json index a332051a..13d0f384 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -16,7 +16,6 @@ "type": "string", "format": "file-path", "exists": true, - "schema": "subworkflows/local/utils_references/schema_references.json", "mimetype": "text/yaml", "pattern": "^\\S+\\.(yaml|yml)$", "errorMessage": "Input datasheet must be an existing file ending in '.yaml' or '.yml' that does not contain spaces", diff --git a/subworkflows/local/utils_nfcore_references_pipeline/main.nf b/subworkflows/local/utils_nfcore_references_pipeline/main.nf index e9dac44b..49f97edc 100644 --- a/subworkflows/local/utils_nfcore_references_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_references_pipeline/main.nf @@ -8,8 +8,6 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { update_references_file } from '../../local/utils_references' - include { checkCondaChannels } from 'plugin/nf-core-utils' include { checkConfigProvided } from 'plugin/nf-core-utils' include { checkProfileProvided } from 'plugin/nf-core-utils' @@ -38,8 +36,6 @@ workflow PIPELINE_INITIALISATION { nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved asset // string: Path to asset yaml file - basepath_final // string: The final basepath to replace in the asset yaml file - basepath_to_replace // array: The basepath to replace in the asset yaml file help // boolean: Display help message and exit help_full // boolean: Show the full help message show_hidden // boolean: Show hidden parameters in the help message @@ -158,7 +154,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from asset file provided through params.asset references = channel.fromList( samplesheetToList( - update_references_file(asset, basepath_final, basepath_to_replace), + asset, "${projectDir}/subworkflows/local/utils_references/schema_references.json", ) ) diff --git a/subworkflows/local/utils_references/main.nf b/subworkflows/local/utils_references/main.nf index e057a32f..ea58927f 100644 --- a/subworkflows/local/utils_references/main.nf +++ b/subworkflows/local/utils_references/main.nf @@ -12,14 +12,12 @@ workflow UTILS_REFERENCES { param_value attribute_file attribute_value - basepath_final - basepath_to_replace main: references = channel.fromList( samplesheetToList( - update_references_file(yaml_reference, basepath_final, basepath_to_replace), + yaml_reference, "${projectDir}/subworkflows/local/utils_references/schema_references.json", ) ) @@ -40,26 +38,6 @@ workflow UTILS_REFERENCES { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// Change all igenomes base parameter values in the file to the correct value -def update_references_file(yaml_reference, basepath_final, basepath_to_replace) { - def correct_yaml_file = file(yaml_reference, checkIfExists: true) - - if (basepath_final) { - def staged_yaml_file = "${workflow.workDir}/tmp/${java.util.UUID.randomUUID().toString()}.${correct_yaml_file.extension}" - correct_yaml_file.copyTo(staged_yaml_file) - correct_yaml_file = file(staged_yaml_file, checkIfExists: true) - - // Use a local variable to accumulate changes - def updated_yaml_content = correct_yaml_file.text - basepath_to_replace.each { basepath_replacement -> - updated_yaml_content = updated_yaml_content.replace(basepath_replacement, basepath_final) - } - correct_yaml_file.text = updated_yaml_content - } - - return correct_yaml_file -} - def get_references_file(references, param, attribute) { return references .map { meta, _readme -> diff --git a/tests/assets/GRCh38_chr21_placeholder.yml b/tests/assets/GRCh38_chr21_placeholder.yml new file mode 100644 index 00000000..99c0dba4 --- /dev/null +++ b/tests/assets/GRCh38_chr21_placeholder.yml @@ -0,0 +1,8 @@ +# Test fixture for ${params.references_base_path} substitution (nf-schema +# validation.allowParamsSubstitution). Placeholder carries no trailing slash; +# the base path carries it. +- genome: GRCh38_chr21 + fasta: ${params.references_base_path}GRCh38_chr21/GRCh38_chr21.fa + mito_name: MT + source: nf-core + species: Homo_sapiens diff --git a/tests/basepath_substitution.nf.test b/tests/basepath_substitution.nf.test new file mode 100644 index 00000000..687eaac1 --- /dev/null +++ b/tests/basepath_substitution.nf.test @@ -0,0 +1,34 @@ +nextflow_pipeline { + + name "Reference base path substitution" + script "../main.nf" + tag "pipeline" + + // Datasheets may carry ${params.references_base_path} placeholders instead of + // hardcoded roots. nf-schema substitutes them at load time + // (validation.allowParamsSubstitution). If substitution stops working the + // literal placeholder reaches file(checkIfExists: true) and this run fails, + // so workflow.success is a real assertion rather than a smoke test. + test('--tools faidx with a datasheet carrying placeholders') { + + options "-output-dir $outputDir" + + when { + params { + input = "${new File('tests/assets/GRCh38_chr21_placeholder.yml').absolutePath}" + outdir = "$outputDir" + tools = 'faidx' + tools_bundle = null + skip_tools = null + references_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/references/references/' + } + } + + then { + assert workflow.success + // The placeholder resolved to a real FAI, not a literal path segment + def fai = getAllFilesFromDir(params.outdir, relative: true, ignore: ['pipeline_info/*']) + assert fai.any { it.endsWith('.fa.fai') } + } + } +} From 5aa1740140c1a80e6cf027f185fa4df2e0f124b1 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 22 Oct 2026 11:30:50 +0200 Subject: [PATCH 2/4] Use projectDir for the substitution test fixture path new File(...).absolutePath was a workaround for an error actually caused by a GString test name; projectDir resolves the same way and reads as intent. Generated by OpenCode --- tests/basepath_substitution.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basepath_substitution.nf.test b/tests/basepath_substitution.nf.test index 687eaac1..21bdbb37 100644 --- a/tests/basepath_substitution.nf.test +++ b/tests/basepath_substitution.nf.test @@ -15,7 +15,7 @@ nextflow_pipeline { when { params { - input = "${new File('tests/assets/GRCh38_chr21_placeholder.yml').absolutePath}" + input = "${projectDir}/tests/assets/GRCh38_chr21_placeholder.yml" outdir = "$outputDir" tools = 'faidx' tools_bundle = null From ae467617034c61c35576752c6dc0e37e69cee2d7 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Oct 2026 09:58:10 +0200 Subject: [PATCH 3/4] Carry the base path slash in the datasheet placeholder The placeholder now owns the leading slash, so the base path value stays clean and the join point stays visible in the datasheet. Add the --references_base_path section to usage.md, and rewrite the schema description and help_text, which still described the prefix rewriting removed by the substitution change. Convert the substitution test to the snapshot style used elsewhere in the suite. Generated by OpenCode --- docs/usage.md | 23 ++++++++++++- nextflow.config | 7 ++-- nextflow_schema.json | 4 +-- tests/assets/GRCh38_chr21_placeholder.yml | 6 ++-- tests/basepath_substitution.nf.test | 26 +++++++++----- tests/basepath_substitution.nf.test.snap | 41 +++++++++++++++++++++++ 6 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 tests/basepath_substitution.nf.test.snap diff --git a/docs/usage.md b/docs/usage.md index c25b84b8..cdf86c20 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -158,7 +158,28 @@ TODO TODO -An [example asset](../assets/asset.yml) has been provided with the pipeline. +Example asset files are published in the [nf-core/references-datasheets](https://github.com/nf-core/references-datasheets) repository. + +### `--references_base_path` + +A datasheet path may carry a `${params.references_base_path}` placeholder instead of a hardcoded root: + +```yaml +- genome: GRCh38_chr21 + fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa +``` + +The pipeline substitutes the parameter into the path as the datasheet loads, so one datasheet can be used against any mirror of the reference files: + +```bash +--references_base_path '/data/mirror/igenomes' +``` + +The default is `s3://ngi-igenomes/igenomes`, the root that published datasheets are written against. With that default the example above resolves to `s3://ngi-igenomes/igenomes/GRCh38_chr21/GRCh38_chr21.fa`. + +The placeholder carries the leading slash, so the value must not end with one. A trailing slash produces a doubled slash in the path. + +Datasheets without placeholders are unchanged and need no edit. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with `--input`. ## Running the pipeline diff --git a/nextflow.config b/nextflow.config index d8284313..31bccf3d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -12,9 +12,7 @@ params { monochrome_logs = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' trace_report_suffix = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') - // Default root substituted for ${params.references_base_path} placeholders in datasheets. - // Unconditional so substitution can never leave a literal placeholder or a null prefix in a path. - references_base_path = 's3://ngi-igenomes/igenomes/' + references_base_path = 's3://ngi-igenomes/igenomes' custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -260,9 +258,8 @@ lineage.enabled = true validation { monochromeLogs = params.monochrome_logs - // Substitute ${params.X} placeholders in datasheet values at load time, replacing - // the previous update_references_file whole-YAML rewrite. allowParamsSubstitution = true } + // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json index 13d0f384..61a60e44 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -56,8 +56,8 @@ "references_base_path": { "type": "string", "format": "directory-path", - "description": "Base path used to resolve reference file paths in the datasheet, replacing the default 's3://ngi-igenomes/igenomes/' prefix.", - "help_text": "Datasheets may carry igenomes-style 's3://ngi-igenomes/igenomes/...' paths. When this parameter is set, that prefix is rewritten to this base path before files are resolved, so you can point at a local or institutional mirror of igenomes without editing the datasheet.", + "description": "Base path substituted for ${params.references_base_path} placeholders in datasheet paths.", + "help_text": "Datasheet file paths may contain a ${params.references_base_path} placeholder instead of a hardcoded root, for example ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa. The placeholder is replaced by this parameter when the datasheet loads, so one datasheet can be used against any mirror of the reference files. The default is s3://ngi-igenomes/igenomes, the root that published datasheets are written against. The placeholder carries the leading slash, so the value must not end with one. Datasheets with no placeholder are unaffected. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with --input.", "fa_icon": "fas fa-ban" }, "email": { diff --git a/tests/assets/GRCh38_chr21_placeholder.yml b/tests/assets/GRCh38_chr21_placeholder.yml index 99c0dba4..bfd4f3ae 100644 --- a/tests/assets/GRCh38_chr21_placeholder.yml +++ b/tests/assets/GRCh38_chr21_placeholder.yml @@ -1,8 +1,8 @@ # Test fixture for ${params.references_base_path} substitution (nf-schema -# validation.allowParamsSubstitution). Placeholder carries no trailing slash; -# the base path carries it. +# validation.allowParamsSubstitution). The placeholder carries the leading +# slash; the base path must not end with one. - genome: GRCh38_chr21 - fasta: ${params.references_base_path}GRCh38_chr21/GRCh38_chr21.fa + fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa mito_name: MT source: nf-core species: Homo_sapiens diff --git a/tests/basepath_substitution.nf.test b/tests/basepath_substitution.nf.test index 21bdbb37..b82ac178 100644 --- a/tests/basepath_substitution.nf.test +++ b/tests/basepath_substitution.nf.test @@ -4,11 +4,6 @@ nextflow_pipeline { script "../main.nf" tag "pipeline" - // Datasheets may carry ${params.references_base_path} placeholders instead of - // hardcoded roots. nf-schema substitutes them at load time - // (validation.allowParamsSubstitution). If substitution stops working the - // literal placeholder reaches file(checkIfExists: true) and this run fails, - // so workflow.success is a real assertion rather than a smoke test. test('--tools faidx with a datasheet carrying placeholders') { options "-output-dir $outputDir" @@ -20,15 +15,28 @@ nextflow_pipeline { tools = 'faidx' tools_bundle = null skip_tools = null - references_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/references/references/' + references_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/references/references' } } then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') assert workflow.success - // The placeholder resolved to a real FAI, not a literal path segment - def fai = getAllFilesFromDir(params.outdir, relative: true, ignore: ['pipeline_info/*']) - assert fai.any { it.endsWith('.fa.fai') } + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key) + removeFromYamlMap("${outputDir}/pipeline_info/nf_core_references_software_mqc_versions.yml", 'Workflow'), + // All stable path name, with a relative path + stable_path, + // All files with stable contents + stable_content + ).match() } + ) } } } diff --git a/tests/basepath_substitution.nf.test.snap b/tests/basepath_substitution.nf.test.snap new file mode 100644 index 00000000..9693af70 --- /dev/null +++ b/tests/basepath_substitution.nf.test.snap @@ -0,0 +1,41 @@ +{ + "--tools faidx with a datasheet carrying placeholders": { + "content": [ + 5, + { + "SAMTOOLS_FAIDX": { + "samtools": "1.24" + } + }, + [ + "Homo_sapiens", + "Homo_sapiens/nf-core", + "Homo_sapiens/nf-core/GRCh38_chr21", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta/GRCh38_chr21.fa.fai", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_references_software_mqc_versions.yml" + ], + [ + "GRCh38_chr21.fa.fai:md5,6ea51e1c1231da5e49212ea9f493d703", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-23T09:24:07.623957777", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.10.0" + } + } +} \ No newline at end of file From 05b6a7ae5c110147c7b1354a969e18c56581a2fb Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Oct 2026 09:58:37 +0200 Subject: [PATCH 4/4] Bump nf-core-utils to 0.5.0 Fixes dumpParametersToJSON, which the utils subworkflow imports, and adds ReferenceSelection with igenomes_base substitution. Generated by OpenCode --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 31bccf3d..749e5eb4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -249,7 +249,7 @@ manifest { // Nextflow plugins plugins { - id 'nf-core-utils@0.4.0' // nf-core-utils is a collection of utilities for Nextflow pipelines + id 'nf-core-utils@0.5.0' // nf-core-utils is a collection of utilities for Nextflow pipelines id 'nf-schema@3.0.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet }