diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c0ef4ef..0d039e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#163](https://github.com/nf-core/references/pull/163) - Update nf-schema to 3.0.0 and migrate `paramsHelp`, `paramsSummaryLog`, and `paramsSummaryMap` to their single-options-map signatures - [#164](https://github.com/nf-core/references/pull/164) - Replace `update_references_file` string rewriting with nf-schema `allowParamsSubstitution`, so datasheets can carry `${params.references_base_path}` placeholders - [#165](https://github.com/nf-core/references/pull/165) - Document `--references_base_path`, multiple genomes, and the asset file format in `docs/usage.md` +- [#166](https://github.com/nf-core/references/pull/166) - Rename the datasheet `source`, `source_version`, and derived `source_vcf` meta fields to `reference_source`, `annotation_version`, and `annotation_source` (#65) ### Removed diff --git a/README.md b/README.md index 41f56777..24bc1cd6 100644 --- a/README.md +++ b/README.md @@ -82,10 +82,10 @@ Datasheets are stored in [references-datasheets](https://github.com/nf-core/refe - genome: GRCh38_chr21 fasta: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/GRCh38_chr21.fa" gtf: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/GRCh38_chr21.gtf" - source_version: "CUSTOM" + annotation_version: "CUSTOM" readme: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/README.md" - source: "nf-core/references" - source_vcf: "GATK_BUNDLE" + reference_source: "nf-core/references" + annotation_source: "GATK_BUNDLE" species: "Homo_sapiens" vcf: "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz" ``` diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a9eabd02..09ce8efb 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -201,8 +201,8 @@ Each output type has its own path declaration using the `file >>` syntax. Output paths follow this structure: ```text -${meta.species}/${meta.source}/${meta.genome}/Sequence/// -${meta.species}/${meta.source}/${meta.genome}/Annotation// +${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/// +${meta.species}/${meta.reference_source}/${meta.genome}/Annotation// ``` When adding a new output, add a corresponding entry to the `output {}` block in `main.nf`. diff --git a/docs/usage.md b/docs/usage.md index ac9f86a7..3b7c697a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -157,12 +157,12 @@ A datasheet is a YAML list with one entry per genome. `genome` is the only requi ```yaml - genome: GRCh38_chr21 species: Homo_sapiens - source: nf-core + reference_source: nf-core fasta: https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/GRCh38_chr21.fa - genome: GRCh38_chr22 species: Homo_sapiens - source: nf-core + reference_source: nf-core fasta: https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta ``` @@ -182,7 +182,7 @@ An entry lists what a genome already provides. Only `genome` is required; the pi ```yaml - genome: GRCh37 species: Homo_sapiens - source: GATK + reference_source: GATK # Genome and annotation fasta: s3://ngi-igenomes/igenomes/Homo_sapiens/GATK/GRCh37/Sequence/WholeGenomeFasta/human_g1k_v37_decoy.fasta diff --git a/main.nf b/main.nf index a98c8d28..d99e569d 100644 --- a/main.nf +++ b/main.nf @@ -60,7 +60,7 @@ workflow NFCORE_REFERENCES { ncbi_download_input = fasta_download_input.to_download .mix(gff_download_input.to_download, gtf_download_input.to_download) - .map { meta, _file -> [meta.source_version, [meta.reference, meta]] } + .map { meta, _file -> [meta.annotation_version, [meta.reference, meta]] } .groupBy() .map { acc, entries -> def ref_types = entries.collect { entry -> entry[0] } @@ -395,137 +395,137 @@ output { } ascat_alleles { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/ascat/" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/ascat/" } } ascat_loci { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/ascat/" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/ascat/" } } ascat_loci_gc { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/ascat/" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/ascat/" } } ascat_loci_rt { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/ascat/" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/ascat/" } } bowtie1_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/BowtieIndex/1.3.1" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/BowtieIndex/1.3.1" } } bowtie2_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/Bowtie2Index/2.5.2" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/Bowtie2Index/2.5.2" } } bwamem1_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/BWAIndex/0.7.18" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/BWAIndex/0.7.18" } } bwamem2_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/BWAmem2Index/2.2.1" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/BWAmem2Index/2.2.1" } } chr_dir { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/chr" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/chr" } } dragmap_hashmap { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/dragmap/1.2.1" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/dragmap/1.2.1" } } fasta { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" } } fasta_dict { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" } } fasta_fai { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" } } fasta_sizes { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/WholeGenomeFasta/${file.fileName}" } } gtf { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Annotation/Genes/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Annotation/Genes/${file.fileName}" } } gff { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Annotation/Genes/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Annotation/Genes/${file.fileName}" } } hisat2_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/Hisat2Index/" + (meta.source_version == 'unknown' ? '' : meta.source_version) + "/2.2.3" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/Hisat2Index/" + (meta.annotation_version == 'unknown' ? '' : meta.annotation_version) + "/2.2.3" } } intervals_bed { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Annotation/intervals/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Annotation/intervals/${file.fileName}" } } kallisto_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/KallistoIndex/" + (meta.source_version == 'unknown' ? '' : meta.source_version) + "/0.52.0/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/KallistoIndex/" + (meta.annotation_version == 'unknown' ? '' : meta.annotation_version) + "/0.52.0/${file.fileName}" } } msisensorpro_list { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Annotation/msisensorpro/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Annotation/msisensorpro/${file.fileName}" } } rsem_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/RSEMIndex/" + (meta.source_version == 'unknown' ? '' : meta.source_version) + "/1.3.1" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/RSEMIndex/" + (meta.annotation_version == 'unknown' ? '' : meta.annotation_version) + "/1.3.1" } } salmon_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/SalmonIndex/" + (meta.source_version == 'unknown' ? '' : meta.source_version) + "/2.7.0" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/SalmonIndex/" + (meta.annotation_version == 'unknown' ? '' : meta.annotation_version) + "/2.7.0" } } snapaligner_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/snap" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/snap" } } splice_sites { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/SpliceSites/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/SpliceSites/${file.fileName}" } } star_index { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/STARIndex/" + (meta.source_version == 'unknown' ? '' : meta.source_version) + "/2.7.11b" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/STARIndex/" + (meta.annotation_version == 'unknown' ? '' : meta.annotation_version) + "/2.7.11b" } } transcript_fasta { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Sequence/TranscriptFasta/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Sequence/TranscriptFasta/${file.fileName}" } } vcf_tbi { path { meta, file -> - file >> "${meta.species}/${meta.source}/${meta.genome}/Annotation/${meta.source_vcf}/${file.fileName}" + file >> "${meta.species}/${meta.reference_source}/${meta.genome}/Annotation/${meta.annotation_source}/${file.fileName}" } } } diff --git a/subworkflows/local/datasheet_to_channel/main.nf b/subworkflows/local/datasheet_to_channel/main.nf index 9e1349ee..55633d05 100644 --- a/subworkflows/local/datasheet_to_channel/main.nf +++ b/subworkflows/local/datasheet_to_channel/main.nf @@ -161,8 +161,8 @@ def reduceMeta(meta_) { record( id: meta_.id, genome: meta_.genome, - source: meta_.source, - source_version: meta_.source_version, + reference_source: meta_.reference_source, + annotation_version: meta_.annotation_version, species: meta_.species, ) } @@ -181,7 +181,7 @@ def toVcfChannel(datasheet, type) { def tbis = !meta[tbi_key] ? [] : meta[tbi_key] instanceof List ? meta[tbi_key] : [meta[tbi_key]] vcfs.withIndex().collect { vcf_, i -> def meta_extra = record(run_tabix: i >= tbis.size() || !tbis[i]) - meta_extra += record(type: type, source_vcf: meta[source_key]) + meta_extra += record(type: type, annotation_source: meta[source_key]) [reduceMeta(meta) + meta_extra, file(vcf_, checkIfExists: true)] } } diff --git a/subworkflows/local/utils_references/schema_references.json b/subworkflows/local/utils_references/schema_references.json index 87809778..61c6cc85 100644 --- a/subworkflows/local/utils_references/schema_references.json +++ b/subworkflows/local/utils_references/schema_references.json @@ -20,18 +20,18 @@ "default": "unknown", "errorMessage": "Website of origin of the reference must be a valid URL or 'unknown'" }, - "source": { - "meta": ["source"], + "reference_source": { + "meta": ["reference_source"], "type": "string", "pattern": "^\\S+$", - "errorMessage": "Source of genome cannot contain spaces" + "errorMessage": "Reference source of the genome cannot contain spaces" }, - "source_version": { - "meta": ["source_version"], + "annotation_version": { + "meta": ["annotation_version"], "type": "string", "pattern": "^\\S+$", "default": "unknown", - "errorMessage": "Source version used to create annotation files (gff/gtf related files) cannot contain spaces" + "errorMessage": "Annotation version used to create annotation files (gff/gtf related files) cannot contain spaces" }, "species": { "meta": ["species"], diff --git a/tests/assets/GRCh38_chr21_placeholder.yml b/tests/assets/GRCh38_chr21_placeholder.yml index bfd4f3ae..8e290499 100644 --- a/tests/assets/GRCh38_chr21_placeholder.yml +++ b/tests/assets/GRCh38_chr21_placeholder.yml @@ -4,5 +4,5 @@ - genome: GRCh38_chr21 fasta: ${params.references_base_path}/GRCh38_chr21/GRCh38_chr21.fa mito_name: MT - source: nf-core + reference_source: nf-core species: Homo_sapiens