diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cf6c392..479054cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,16 +65,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#157](https://github.com/nf-core/references/pull/157) - Replace `.out` channel access with typed workflow return assignment - [#161](https://github.com/nf-core/references/pull/161) - Document `references_base_path` description and igenomes prefix-rewriting behaviour - [#161](https://github.com/nf-core/references/pull/161) - Replace placeholder `TODO` error messages in `schema_references.json` +- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow - [#162](https://github.com/nf-core/references/pull/162) - Resolve datasheet `splice_sites` paths to files so publishing works on Nextflow 26.10 - [#163](https://github.com/nf-core/references/pull/163) - Update nf-schema to 3.0.0 and migrate `paramsHelp`, `paramsSummaryLog`, and `paramsSummaryMap` to their single-options-map signatures -- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow +- [#164](https://github.com/nf-core/references/pull/164) - Replace `update_references_file` string rewriting with nf-schema `allowParamsSubstitution`, so datasheets can carry `${params.references_base_path}` placeholders ### Removed -- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome` - - [#146](https://github.com/nf-core/references/pull/146) - Removed `index.json` output in favor of Nextflow lineage tracking +- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome` + ### Fixed - [#77](https://github.com/nf-core/references/pull/77) - Fix createsequencedictionary usage diff --git a/docs/usage.md b/docs/usage.md index c25b84b8..cdf86c20 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -158,7 +158,28 @@ TODO TODO -An [example asset](../assets/asset.yml) has been provided with the pipeline. +Example asset files are published in the [nf-core/references-datasheets](https://github.com/nf-core/references-datasheets) repository. + +### `--references_base_path` + +A datasheet path may carry a `${params.references_base_path}` placeholder instead of a hardcoded root: + +```yaml +- genome: GRCh38_chr21 + fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa +``` + +The pipeline substitutes the parameter into the path as the datasheet loads, so one datasheet can be used against any mirror of the reference files: + +```bash +--references_base_path '/data/mirror/igenomes' +``` + +The default is `s3://ngi-igenomes/igenomes`, the root that published datasheets are written against. With that default the example above resolves to `s3://ngi-igenomes/igenomes/GRCh38_chr21/GRCh38_chr21.fa`. + +The placeholder carries the leading slash, so the value must not end with one. A trailing slash produces a doubled slash in the path. + +Datasheets without placeholders are unchanged and need no edit. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with `--input`. ## Running the pipeline diff --git a/main.nf b/main.nf index 488b3968..a98c8d28 100644 --- a/main.nf +++ b/main.nf @@ -295,8 +295,6 @@ workflow { params.help, params.help_full, params.show_hidden, - params.references_base_path, - ['s3://ngi-igenomes/igenomes/'], tools, ) diff --git a/nextflow.config b/nextflow.config index 065725c5..749e5eb4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -12,6 +12,7 @@ params { monochrome_logs = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' trace_report_suffix = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') + references_base_path = 's3://ngi-igenomes/igenomes' custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -248,7 +249,7 @@ manifest { // Nextflow plugins plugins { - id 'nf-core-utils@0.4.0' // nf-core-utils is a collection of utilities for Nextflow pipelines + id 'nf-core-utils@0.5.0' // nf-core-utils is a collection of utilities for Nextflow pipelines id 'nf-schema@3.0.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } @@ -257,6 +258,8 @@ lineage.enabled = true validation { monochromeLogs = params.monochrome_logs + allowParamsSubstitution = true } + // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json index a332051a..61a60e44 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -16,7 +16,6 @@ "type": "string", "format": "file-path", "exists": true, - "schema": "subworkflows/local/utils_references/schema_references.json", "mimetype": "text/yaml", "pattern": "^\\S+\\.(yaml|yml)$", "errorMessage": "Input datasheet must be an existing file ending in '.yaml' or '.yml' that does not contain spaces", @@ -57,8 +56,8 @@ "references_base_path": { "type": "string", "format": "directory-path", - "description": "Base path used to resolve reference file paths in the datasheet, replacing the default 's3://ngi-igenomes/igenomes/' prefix.", - "help_text": "Datasheets may carry igenomes-style 's3://ngi-igenomes/igenomes/...' paths. When this parameter is set, that prefix is rewritten to this base path before files are resolved, so you can point at a local or institutional mirror of igenomes without editing the datasheet.", + "description": "Base path substituted for ${params.references_base_path} placeholders in datasheet paths.", + "help_text": "Datasheet file paths may contain a ${params.references_base_path} placeholder instead of a hardcoded root, for example ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa. The placeholder is replaced by this parameter when the datasheet loads, so one datasheet can be used against any mirror of the reference files. The default is s3://ngi-igenomes/igenomes, the root that published datasheets are written against. The placeholder carries the leading slash, so the value must not end with one. Datasheets with no placeholder are unaffected. This parameter only rewrites paths inside the datasheet; the location of the datasheet itself is set with --input.", "fa_icon": "fas fa-ban" }, "email": { diff --git a/subworkflows/local/utils_nfcore_references_pipeline/main.nf b/subworkflows/local/utils_nfcore_references_pipeline/main.nf index e9dac44b..49f97edc 100644 --- a/subworkflows/local/utils_nfcore_references_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_references_pipeline/main.nf @@ -8,8 +8,6 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { update_references_file } from '../../local/utils_references' - include { checkCondaChannels } from 'plugin/nf-core-utils' include { checkConfigProvided } from 'plugin/nf-core-utils' include { checkProfileProvided } from 'plugin/nf-core-utils' @@ -38,8 +36,6 @@ workflow PIPELINE_INITIALISATION { nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved asset // string: Path to asset yaml file - basepath_final // string: The final basepath to replace in the asset yaml file - basepath_to_replace // array: The basepath to replace in the asset yaml file help // boolean: Display help message and exit help_full // boolean: Show the full help message show_hidden // boolean: Show hidden parameters in the help message @@ -158,7 +154,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from asset file provided through params.asset references = channel.fromList( samplesheetToList( - update_references_file(asset, basepath_final, basepath_to_replace), + asset, "${projectDir}/subworkflows/local/utils_references/schema_references.json", ) ) diff --git a/subworkflows/local/utils_references/main.nf b/subworkflows/local/utils_references/main.nf index e057a32f..ea58927f 100644 --- a/subworkflows/local/utils_references/main.nf +++ b/subworkflows/local/utils_references/main.nf @@ -12,14 +12,12 @@ workflow UTILS_REFERENCES { param_value attribute_file attribute_value - basepath_final - basepath_to_replace main: references = channel.fromList( samplesheetToList( - update_references_file(yaml_reference, basepath_final, basepath_to_replace), + yaml_reference, "${projectDir}/subworkflows/local/utils_references/schema_references.json", ) ) @@ -40,26 +38,6 @@ workflow UTILS_REFERENCES { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// Change all igenomes base parameter values in the file to the correct value -def update_references_file(yaml_reference, basepath_final, basepath_to_replace) { - def correct_yaml_file = file(yaml_reference, checkIfExists: true) - - if (basepath_final) { - def staged_yaml_file = "${workflow.workDir}/tmp/${java.util.UUID.randomUUID().toString()}.${correct_yaml_file.extension}" - correct_yaml_file.copyTo(staged_yaml_file) - correct_yaml_file = file(staged_yaml_file, checkIfExists: true) - - // Use a local variable to accumulate changes - def updated_yaml_content = correct_yaml_file.text - basepath_to_replace.each { basepath_replacement -> - updated_yaml_content = updated_yaml_content.replace(basepath_replacement, basepath_final) - } - correct_yaml_file.text = updated_yaml_content - } - - return correct_yaml_file -} - def get_references_file(references, param, attribute) { return references .map { meta, _readme -> diff --git a/tests/assets/GRCh38_chr21_placeholder.yml b/tests/assets/GRCh38_chr21_placeholder.yml new file mode 100644 index 00000000..bfd4f3ae --- /dev/null +++ b/tests/assets/GRCh38_chr21_placeholder.yml @@ -0,0 +1,8 @@ +# Test fixture for ${params.references_base_path} substitution (nf-schema +# validation.allowParamsSubstitution). The placeholder carries the leading +# slash; the base path must not end with one. +- genome: GRCh38_chr21 + fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa + mito_name: MT + source: nf-core + species: Homo_sapiens diff --git a/tests/basepath_substitution.nf.test b/tests/basepath_substitution.nf.test new file mode 100644 index 00000000..b82ac178 --- /dev/null +++ b/tests/basepath_substitution.nf.test @@ -0,0 +1,42 @@ +nextflow_pipeline { + + name "Reference base path substitution" + script "../main.nf" + tag "pipeline" + + test('--tools faidx with a datasheet carrying placeholders') { + + options "-output-dir $outputDir" + + when { + params { + input = "${projectDir}/tests/assets/GRCh38_chr21_placeholder.yml" + outdir = "$outputDir" + tools = 'faidx' + tools_bundle = null + skip_tools = null + references_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/references/references' + } + } + + then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow and pipeline versions are removed (all from the workflow key) + removeFromYamlMap("${outputDir}/pipeline_info/nf_core_references_software_mqc_versions.yml", 'Workflow'), + // All stable path name, with a relative path + stable_path, + // All files with stable contents + stable_content + ).match() } + ) + } + } +} diff --git a/tests/basepath_substitution.nf.test.snap b/tests/basepath_substitution.nf.test.snap new file mode 100644 index 00000000..9693af70 --- /dev/null +++ b/tests/basepath_substitution.nf.test.snap @@ -0,0 +1,41 @@ +{ + "--tools faidx with a datasheet carrying placeholders": { + "content": [ + 5, + { + "SAMTOOLS_FAIDX": { + "samtools": "1.24" + } + }, + [ + "Homo_sapiens", + "Homo_sapiens/nf-core", + "Homo_sapiens/nf-core/GRCh38_chr21", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta", + "Homo_sapiens/nf-core/GRCh38_chr21/Sequence/WholeGenomeFasta/GRCh38_chr21.fa.fai", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_references_software_mqc_versions.yml" + ], + [ + "GRCh38_chr21.fa.fai:md5,6ea51e1c1231da5e49212ea9f493d703", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-23T09:24:07.623957777", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.10.0" + } + } +} \ No newline at end of file