diff --git a/CHANGELOG.md b/CHANGELOG.md index 97312acb..bbc4125b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#154](https://github.com/nf-core/references/pull/154) - Added `--hisat2_skip_splice_sites` parameter to skip splice site extraction for prokaryotic genomes - [#155](https://github.com/nf-core/references/pull/155) - Added species-specific STAR index parameters (`--star_sjdbOverhang`, `--star_genomeSAindexNbases`) with per-genome asset file configuration - [#159](https://github.com/nf-core/references/pull/159) - Add release roadmap for rnaseq and sarek consumability +- [#160](https://github.com/nf-core/references/pull/160) - Support list-valued VCF fields in datasheets, with positional TBI pairing (null entries trigger index building) and missing `ascat_loci_rt` schema entry ### Changed diff --git a/subworkflows/local/datasheet_to_channel/main.nf b/subworkflows/local/datasheet_to_channel/main.nf index 3db1b6b9..b1a57108 100644 --- a/subworkflows/local/datasheet_to_channel/main.nf +++ b/subworkflows/local/datasheet_to_channel/main.nf @@ -109,46 +109,18 @@ workflow DATASHEET_TO_CHANNEL { } // HANDLING OF VCF + // VCF fields can be a single path or a list of paths, with an optional + // parallel list of indexes (null entries mean the index needs to be built) - dbsnp = datasheet - .filter { meta, _readme -> meta.vcf_dbsnp_vcf } - .map { meta, _readme -> - def meta_extra = record(run_tabix: !meta.vcf_dbsnp_vcf_tbi) - meta_extra += record(type: 'dbsnp', source_vcf: meta.vcf_dbsnp_vcf_source) - [reduceMeta(meta) + meta_extra, files(meta.vcf_dbsnp_vcf, checkIfExists: true)] - } + dbsnp = toVcfChannel(datasheet, 'dbsnp') - germline_resource = datasheet - .filter { meta, _readme -> meta.vcf_germline_resource_vcf } - .map { meta, _readme -> - def meta_extra = record(run_tabix: !meta.vcf_germline_resource_vcf_tbi) - meta_extra += record(type: 'germline_resource', source_vcf: meta.vcf_germline_resource_vcf_source) - [reduceMeta(meta) + meta_extra, file(meta.vcf_germline_resource_vcf, checkIfExists: true)] - } + germline_resource = toVcfChannel(datasheet, 'germline_resource') - known_indels = datasheet - .filter { meta, _readme -> meta.vcf_known_indels_vcf } - .map { meta, _readme -> - def meta_extra = record(run_tabix: !meta.vcf_known_indels_vcf_tbi) - meta_extra += record(type: 'known_indels', source_vcf: meta.vcf_known_indels_vcf_source) - [reduceMeta(meta) + meta_extra, file(meta.vcf_known_indels_vcf, checkIfExists: true)] - } + known_indels = toVcfChannel(datasheet, 'known_indels') - known_snps = datasheet - .filter { meta, _readme -> meta.vcf_known_snps_vcf } - .map { meta, _readme -> - def meta_extra = record(run_tabix: !meta.vcf_known_snps_vcf_tbi) - meta_extra += record(type: 'known_snps', source_vcf: meta.vcf_known_snps_vcf_source) - [reduceMeta(meta) + meta_extra, file(meta.vcf_known_snps_vcf, checkIfExists: true)] - } + known_snps = toVcfChannel(datasheet, 'known_snps') - pon = datasheet - .filter { meta, _readme -> meta.vcf_pon_vcf } - .map { meta, _readme -> - def meta_extra = record(run_tabix: !meta.vcf_pon_vcf_tbi) - meta_extra += record(type: 'pon', source_vcf: meta.vcf_pon_vcf_source) - [reduceMeta(meta) + meta_extra, file(meta.vcf_pon_vcf, checkIfExists: true)] - } + pon = toVcfChannel(datasheet, 'pon') vcf = channel.empty() .mix( @@ -158,7 +130,6 @@ workflow DATASHEET_TO_CHANNEL { known_snps, pon, ) - .flatMap { meta, vcfs -> vcfs instanceof Path ? [[meta, vcfs]] : vcfs.toList().collect { vcf_ -> [meta, vcf_] } } emit: ascat_alleles = ascat_alleles // channel: [meta, *.ascat_alleles.txt] @@ -195,3 +166,23 @@ def reduceMeta(meta_) { species: meta_.species, ) } + +// Build a [meta, vcf] channel for one VCF type +// The datasheet field can hold a single path or a list of paths +// Indexes are paired positionally, a null or missing entry means the index has to be built +def toVcfChannel(datasheet, type) { + def vcf_key = "vcf_${type}_vcf" + def tbi_key = "vcf_${type}_vcf_tbi" + def source_key = "vcf_${type}_vcf_source" + datasheet + .filter { meta, _readme -> meta[vcf_key] } + .flatMap { meta, _readme -> + def vcfs = meta[vcf_key] instanceof List ? meta[vcf_key] : [meta[vcf_key]] + def tbis = !meta[tbi_key] ? [] : meta[tbi_key] instanceof List ? meta[tbi_key] : [meta[tbi_key]] + vcfs.withIndex().collect { vcf_, i -> + def meta_extra = record(run_tabix: i >= tbis.size() || !tbis[i]) + meta_extra += record(type: type, source_vcf: meta[source_key]) + [reduceMeta(meta) + meta_extra, file(vcf_, checkIfExists: true)] + } + } +} diff --git a/subworkflows/local/utils_references/schema_references.json b/subworkflows/local/utils_references/schema_references.json index 46b80807..166dee9d 100644 --- a/subworkflows/local/utils_references/schema_references.json +++ b/subworkflows/local/utils_references/schema_references.json @@ -57,6 +57,12 @@ "pattern": "^\\S+$", "errorMessage": "TODO" }, + "ascat_loci_rt": { + "meta": ["ascat_loci_rt"], + "type": "string", + "pattern": "^\\S+$", + "errorMessage": "TODO" + }, "bed12": { "meta": ["bed12"], "type": "string", @@ -255,17 +261,48 @@ }, "vcf_dbsnp_vcf": { "meta": ["vcf_dbsnp_vcf"], - "type": "string", - "exists": true, - "pattern": "^\\S+\\.vcf(\\.gz)?$", - "errorMessage": "DBSNP VCF must be an existing file ending in '.vcf' that does not contain spaces. This file can be bgzipped" + "anyOf": [ + { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + }, + { + "type": "array", + "items": { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + } + } + ], + "errorMessage": "DBSNP VCF must be an existing file (or list of existing files) ending in '.vcf' that does not contain spaces. This file can be bgzipped" }, "vcf_dbsnp_vcf_tbi": { "meta": ["vcf_dbsnp_vcf_tbi"], - "type": "string", - "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", - "exists": true, - "errorMessage": "DBSNP VCF index must be an existing file ending in '.vcf.gz.tbi' that does not contain spaces" + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "null" + } + ] + } + } + ], + "errorMessage": "DBSNP VCF index must be an existing file (or list of existing files, null entries allowed) ending in '.vcf.gz.tbi' that does not contain spaces" }, "vcf_dbsnp_vcf_vqsr": { "meta": ["vcf_dbsnp_vcf_vqsr"], @@ -281,17 +318,48 @@ }, "vcf_germline_resource_vcf": { "meta": ["vcf_germline_resource_vcf"], - "type": "string", - "exists": true, - "pattern": "^\\S+\\.vcf(\\.gz)?$", - "errorMessage": "Germline Resource VCF must be an existing file ending in '.vcf' that does not contain spaces. This file can be bgzipped" + "anyOf": [ + { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + }, + { + "type": "array", + "items": { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + } + } + ], + "errorMessage": "Germline Resource VCF must be an existing file (or list of existing files) ending in '.vcf' that does not contain spaces. This file can be bgzipped" }, "vcf_germline_resource_vcf_tbi": { "meta": ["vcf_germline_resource_vcf_tbi"], - "type": "string", - "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", - "exists": true, - "errorMessage": "Germline Resource VCF index must be an existing file ending in '.vcf.gz.tbi' that does not contain spaces" + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "null" + } + ] + } + } + ], + "errorMessage": "Germline Resource VCF index must be an existing file (or list of existing files, null entries allowed) ending in '.vcf.gz.tbi' that does not contain spaces" }, "vcf_germline_resource_vcf_source": { "meta": ["vcf_germline_resource_vcf_source"], @@ -301,17 +369,48 @@ }, "vcf_known_indels_vcf": { "meta": ["vcf_known_indels_vcf"], - "type": "string", - "exists": true, - "pattern": "^\\S+\\.vcf(\\.gz)?$", - "errorMessage": "Known indels VCF must be an existing file ending in '.vcf' that does not contain spaces. This file can be bgzipped" + "anyOf": [ + { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + }, + { + "type": "array", + "items": { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + } + } + ], + "errorMessage": "Known indels VCF must be an existing file (or list of existing files) ending in '.vcf' that does not contain spaces. This file can be bgzipped" }, "vcf_known_indels_vcf_tbi": { "meta": ["vcf_known_indels_vcf_tbi"], - "type": "string", - "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", - "exists": true, - "errorMessage": "Known indels VCF index must be an existing file ending in '.vcf.gz.tbi' that does not contain spaces" + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "null" + } + ] + } + } + ], + "errorMessage": "Known indels VCF index must be an existing file (or list of existing files, null entries allowed) ending in '.vcf.gz.tbi' that does not contain spaces" }, "vcf_known_indels_vcf_source": { "meta": ["vcf_known_indels_vcf_source"], @@ -321,17 +420,48 @@ }, "vcf_known_snps_vcf": { "meta": ["vcf_known_snps_vcf"], - "type": "string", - "exists": true, - "pattern": "^\\S+\\.vcf(\\.gz)?$", - "errorMessage": "Known snps VCF must be an existing file ending in '.vcf' that does not contain spaces. This file can be bgzipped" + "anyOf": [ + { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + }, + { + "type": "array", + "items": { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + } + } + ], + "errorMessage": "Known snps VCF must be an existing file (or list of existing files) ending in '.vcf' that does not contain spaces. This file can be bgzipped" }, "vcf_known_snps_vcf_tbi": { "meta": ["vcf_known_snps_vcf_tbi"], - "type": "string", - "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", - "exists": true, - "errorMessage": "Known snps VCF index must be an existing file ending in '.vcf.gz.tbi' that does not contain spaces" + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "null" + } + ] + } + } + ], + "errorMessage": "Known snps VCF index must be an existing file (or list of existing files, null entries allowed) ending in '.vcf.gz.tbi' that does not contain spaces" }, "vcf_known_snps_vcf_source": { "meta": ["vcf_known_snps_vcf_source"], @@ -341,17 +471,48 @@ }, "vcf_pon_vcf": { "meta": ["vcf_pon_vcf"], - "type": "string", - "exists": true, - "pattern": "^\\S+\\.vcf(\\.gz)?$", - "errorMessage": "PON VCF must be an existing file ending in '.vcf' that does not contain spaces. This file can be bgzipped" + "anyOf": [ + { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + }, + { + "type": "array", + "items": { + "type": "string", + "exists": true, + "pattern": "^\\S+\\.vcf(\\.gz)?$" + } + } + ], + "errorMessage": "PON VCF must be an existing file (or list of existing files) ending in '.vcf' that does not contain spaces. This file can be bgzipped" }, "vcf_pon_vcf_tbi": { "meta": ["vcf_pon_vcf_tbi"], - "type": "string", - "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", - "exists": true, - "errorMessage": "PON VCF index must be an existing file ending in '.vcf.gz.tbi' that does not contain spaces" + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.vcf\\.gz\\.tbi?$", + "exists": true + }, + { + "type": "null" + } + ] + } + } + ], + "errorMessage": "PON VCF index must be an existing file (or list of existing files, null entries allowed) ending in '.vcf.gz.tbi' that does not contain spaces" }, "vcf_pon_vcf_source": { "meta": ["vcf_pon_vcf_source"], diff --git a/tests/multiple.nf.test b/tests/multiple.nf.test index 3cc9efcb..33cc9eca 100644 --- a/tests/multiple.nf.test +++ b/tests/multiple.nf.test @@ -39,13 +39,13 @@ nextflow_pipeline { } } - test("-profile test --tools tabix --input references-datasheets/latest/genomes_source/Homo_sapiens/GATK/GRCh37_GRCh38_multiple_dbsnps_globs.yml") { + test("-profile test --tools tabix --input references-datasheets/latest/igenomes/GATK/GRCh37decoy.yml") { options "-output-dir $outputDir" when { params { - input = "https://raw.githubusercontent.com/nf-core/references-datasheets/latest/genomes_source/Homo_sapiens/GATK/GRCh37_GRCh38_multiple_dbsnps_globs.yml" + input = "https://raw.githubusercontent.com/nf-core/references-datasheets/latest/igenomes/GATK/GRCh37decoy.yml" outdir = "$outputDir" tools = 'tabix' tools_bundle = null diff --git a/tests/multiple.nf.test.snap b/tests/multiple.nf.test.snap index 4736d2ab..b4e71ded 100644 --- a/tests/multiple.nf.test.snap +++ b/tests/multiple.nf.test.snap @@ -88,9 +88,9 @@ "nextflow": "26.04.6" } }, - "-profile test --tools tabix --input references-datasheets/latest/genomes_source/Homo_sapiens/GATK/GRCh37_GRCh38_multiple_dbsnps_globs.yml": { + "-profile test --tools tabix --input references-datasheets/latest/igenomes/GATK/GRCh37decoy.yml": { "content": [ - 8, + 5, { "HTSLIB_BGZIPTABIX": { "htslib": "1.24", @@ -102,12 +102,8 @@ "Homo_sapiens/GATK", "Homo_sapiens/GATK/GRCh37", "Homo_sapiens/GATK/GRCh37/Annotation", - "Homo_sapiens/GATK/GRCh37/Annotation/GATK_BUNDLE", - "Homo_sapiens/GATK/GRCh37/Annotation/GATK_BUNDLE/GRCh37.vcf.gz.tbi", - "Homo_sapiens/GATK/GRCh38", - "Homo_sapiens/GATK/GRCh38/Annotation", - "Homo_sapiens/GATK/GRCh38/Annotation/GATK_BUNDLE", - "Homo_sapiens/GATK/GRCh38/Annotation/GATK_BUNDLE/GRCh38.vcf.gz.tbi", + "Homo_sapiens/GATK/GRCh37/Annotation/GATKBundle", + "Homo_sapiens/GATK/GRCh37/Annotation/GATKBundle/GRCh37.vcf.gz.tbi", "multiqc", "multiqc/multiqc_data", "multiqc/multiqc_data/llms-full.txt", @@ -125,10 +121,10 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-08-14T15:37:44.713192869", + "timestamp": "2026-10-13T13:29:40.583645032", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.6" + "nextflow": "26.04.7" } } } \ No newline at end of file