Merge prepare_genome subworkflows into a single prepare_genome

CHANGELOG.md

@@ -65,9 +65,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#157](https://github.com/nf-core/references/pull/157) - Replace `.out` channel access with typed workflow return assignment
 - [#161](https://github.com/nf-core/references/pull/161) - Document `references_base_path` description and igenomes prefix-rewriting behaviour
 - [#161](https://github.com/nf-core/references/pull/161) - Replace placeholder `TODO` error messages in `schema_references.json`
+- [#162](https://github.com/nf-core/references/pull/162) - Resolve datasheet `splice_sites` paths to files so publishing works on Nextflow 26.10
+- [#162](https://github.com/nf-core/references/pull/162) - Merge `prepare_genome_dnaseq` and `prepare_genome_rnaseq` into a single `prepare_genome` subworkflow
 
 ### Removed
 
+- [#162](https://github.com/nf-core/references/pull/162) - Removed the separate `prepare_genome_dnaseq` and `prepare_genome_rnaseq` subworkflows in favour of `prepare_genome`
+
 - [#146](https://github.com/nf-core/references/pull/146) - Removed `index.json` output in favor of Nextflow lineage tracking
 
 ### Fixed

subworkflows/local/datasheet_to_channel/main.nf

@@ -95,7 +95,7 @@ workflow DATASHEET_TO_CHANNEL {
 
     splice_sites = datasheet
         .filter { meta, _readme -> meta.splice_sites }
-        .map { meta, _readme -> [reduceMeta(meta), meta.splice_sites] }
+        .map { meta, _readme -> [reduceMeta(meta), file(meta.splice_sites, checkIfExists: true)] }
 
     transcript_fasta = datasheet
         .filter { meta, _readme -> meta.transcript_fasta }

subworkflows/local/prepare_genome_rnaseq/main.nf → subworkflows/local/prepare_genome/main.nf

@@ -1,46 +1,56 @@
 include { BOWTIE2_BUILD                                         } from '../../../modules/nf-core/bowtie2/build'
 include { BOWTIE_BUILD as BOWTIE1_BUILD                         } from '../../../modules/nf-core/bowtie/build'
+include { BWAMEM2_INDEX                                         } from '../../../modules/nf-core/bwamem2/index'
+include { BWA_INDEX as BWAMEM1_INDEX                            } from '../../../modules/nf-core/bwa/index'
+include { DRAGMAP_HASHTABLE                                     } from '../../../modules/nf-core/dragmap/hashtable'
+include { GATK4_CREATESEQUENCEDICTIONARY                        } from '../../../modules/nf-core/gatk4/createsequencedictionary'
+include { GAWK as BUILD_INTERVALS                               } from '../../../modules/nf-core/gawk'
 include { GFFREAD                                               } from '../../../modules/nf-core/gffread'
 include { HISAT2_BUILD                                          } from '../../../modules/nf-core/hisat2/build'
 include { HISAT2_EXTRACTSPLICESITES                             } from '../../../modules/nf-core/hisat2/extractsplicesites'
+include { HTSLIB_BGZIPTABIX                                     } from '../../../modules/nf-core/htslib/bgziptabix'
 include { KALLISTO_INDEX                                        } from '../../../modules/nf-core/kallisto/index'
+include { MSISENSORPRO_SCAN                                     } from '../../../modules/nf-core/msisensorpro/scan'
 include { RSEM_PREPAREREFERENCE as MAKE_TRANSCRIPTS_FASTA       } from '../../../modules/nf-core/rsem/preparereference'
 include { RSEM_PREPAREREFERENCE as RSEM_PREPAREREFERENCE_GENOME } from '../../../modules/nf-core/rsem/preparereference'
 include { SALMON_INDEX                                          } from '../../../modules/nf-core/salmon/index'
 include { SAMTOOLS_FAIDX                                        } from '../../../modules/nf-core/samtools/faidx'
+include { SNAPALIGNER_INDEX                                     } from '../../../modules/nf-core/snapaligner/index'
 include { STAR_GENOMEGENERATE                                   } from '../../../modules/nf-core/star/genomegenerate'
 
-workflow PREPARE_GENOME_RNASEQ {
+workflow PREPARE_GENOME {
     take:
     fasta // channel: [meta, fasta]
     fasta_fai // channel: [meta, fasta_fai]
     gff // channel: [meta, gff]
     gtf // channel: [meta, gtf]
     splice_sites // channel: [meta, splice_sites]
     transcript_fasta // channel: [meta, transcript_fasta]
+    vcf // channel: [meta, vcf]
+    altliftoverfile // channel: altliftoverfile
     tools // List: tools to build references for
     hisat2_build_memory
     hisat2_skip_splice_sites
 
     main:
-
     run_faidx = 'faidx' in tools
     run_sizes = 'sizes' in tools
 
+    // Aligner indexes
     ch_bowtie1_build = BOWTIE1_BUILD(fasta.filter { meta, _fasta -> 'bowtie1' in tools && meta.run_bowtie1 })
 
     ch_bowtie2_build = BOWTIE2_BUILD(fasta.filter { meta, _fasta -> 'bowtie2' in tools && meta.run_bowtie2 })
 
-    ch_samtools_faidx = SAMTOOLS_FAIDX(fasta.filter { meta, _fasta -> (run_faidx || run_sizes) && meta.run_faidx }.map { meta, fasta_ -> [meta, fasta_, []] }, run_sizes)
+    ch_bwamem1_index = BWAMEM1_INDEX(fasta.filter { meta, _fasta -> 'bwamem1' in tools && meta.run_bwamem1 })
 
-    fasta_fai = fasta_fai.mix(ch_samtools_faidx.fai)
+    ch_bwamem2_index = BWAMEM2_INDEX(fasta.filter { meta, _fasta -> 'bwamem2' in tools && meta.run_bwamem2 })
+
+    ch_dragmap_hashtable = DRAGMAP_HASHTABLE(fasta.filter { meta, _fasta -> 'dragmap' in tools && meta.run_dragmap })
 
+    // Annotation and transcript resources
     ch_gffread = GFFREAD(join_by_meta_id(fasta, gff.filter { meta, _gff -> ('hisat2' in tools || 'kallisto' in tools || 'rsem' in tools || 'rsem_make_transcript_fasta' in tools || 'salmon' in tools || 'star' in tools) && meta.run_gffread }))
 
-    gtf = gtf
-        .mix(ch_gffread.gtf)
-        .groupBy()
-        .map { meta, gtf_ -> gtf_[1] ? [meta, gtf_[1]] : [meta, gtf_[0]] }
+    gtf = gtf.mix(ch_gffread.gtf).groupBy().map { meta, gtf_ -> gtf_[1] ? [meta, gtf_[1]] : [meta, gtf_[0]] }
 
     ch_hisat2_extractsplicesites = HISAT2_EXTRACTSPLICESITES(gtf.filter { meta, _gtf -> ('hisat2' in tools || 'hisat2_extractsplicesites' in tools) && meta.run_hisat2 && !hisat2_skip_splice_sites })
 
@@ -60,26 +70,59 @@ workflow PREPARE_GENOME_RNASEQ {
 
     ch_star_genomegenerate = STAR_GENOMEGENERATE(join_by_meta_id(fasta.filter { meta, _fasta -> 'star' in tools && meta.run_star }, gtf))
 
+    // Sequence dictionaries, indexes, and intervals
+    ch_gatk4_dict = GATK4_CREATESEQUENCEDICTIONARY(fasta.filter { meta, _fasta -> 'createsequencedictionary' in tools && meta.run_createsequencedictionary })
+
+    ch_samtools_faidx = SAMTOOLS_FAIDX(fasta.filter { meta, _fasta -> (run_faidx || run_sizes) && meta.run_faidx }.map { meta, fasta_ -> [meta, fasta_, []] }, run_sizes)
+
+    fasta_fai = fasta_fai.mix(ch_samtools_faidx.fai)
+
+    ch_build_intervals = BUILD_INTERVALS(fasta_fai.filter { meta, _fasta_fai -> 'intervals' in tools && meta.run_intervals }, [], false)
+
+    ch_msisensorpro_scan = MSISENSORPRO_SCAN(fasta.filter { meta, _fasta -> 'msisensorpro' in tools && meta.run_msisensorpro })
+
+    ch_bgziptabix = HTSLIB_BGZIPTABIX(
+        vcf.filter { meta, _vcf -> 'tabix' in tools && meta.run_tabix }.map { meta, vcf_ -> [meta, vcf_, null, null] },
+        "compress",
+        true,
+        "vcf",
+    )
+
+    def fasta_for_snapaligner = altliftoverfile
+        ? fasta.combine(altliftoverfile).map { meta, fasta_, altliftoverfile_ -> [meta, fasta_, [], [], altliftoverfile_] }
+        : fasta.map { meta, fasta_ -> [meta, fasta_, [], [], []] }
+
+    ch_snapaligner_index = SNAPALIGNER_INDEX(fasta_for_snapaligner.filter { meta, _fasta, _altcontigfile, _nonaltcontigfile, _altliftoverfile -> 'snapaligner' in tools && meta.run_snapaligner })
+
     emit:
-    bowtie1_index    = ch_bowtie1_build.index // channel: [meta, BowtieIndex/]
-    bowtie2_index    = ch_bowtie2_build.index // channel: [meta, Bowtie2Index/]
+    bowtie1_index     = ch_bowtie1_build.index // channel: [meta, BowtieIndex/]
+    bowtie2_index     = ch_bowtie2_build.index // channel: [meta, Bowtie2Index/]
+    bwamem1_index     = ch_bwamem1_index.index // channel: [meta, BWAmemIndex/]
+    bwamem2_index     = ch_bwamem2_index.index // channel: [meta, BWAmem2memIndex/]
+    dragmap_hashmap   = ch_dragmap_hashtable.hashmap // channel: [meta, DragmapHashtable/]
+    fasta_dict        = ch_gatk4_dict.dict // channel: [meta, *.fa(sta).dict]
     fasta_fai // channel: [meta, *.fa(sta).fai]
-    fasta_sizes      = ch_samtools_faidx.sizes // channel: [meta, *.fa(sta).sizes]
+    fasta_sizes       = ch_samtools_faidx.sizes // channel: [meta, *.fa(sta).sizes]
     gtf // channel: [meta, gtf]
-    hisat2_index     = ch_hisat2_build.index // channel: [meta, Hisat2Index/]
-    kallisto_index   = ch_kallisto_index.index // channel: [meta, KallistoIndex]
-    rsem_index       = ch_rsem_preparereference_genome.index // channel: [meta, RSEMIndex/]
-    salmon_index     = ch_salmon_index.index // channel: [meta, SalmonIndex/]
+    hisat2_index      = ch_hisat2_build.index // channel: [meta, Hisat2Index/]
+    intervals_bed     = ch_build_intervals.output // channel: [meta, *.bed]
+    kallisto_index    = ch_kallisto_index.index // channel: [meta, KallistoIndex]
+    msisensorpro_list = ch_msisensorpro_scan.list // channel: [meta, *.list]
+    rsem_index        = ch_rsem_preparereference_genome.index // channel: [meta, RSEMIndex/]
+    salmon_index      = ch_salmon_index.index // channel: [meta, SalmonIndex/]
+    snapaligner_index = ch_snapaligner_index.index // channel: [meta, snap/]
     splice_sites // channel: [meta, *.splice_sites.txt]
-    star_index       = ch_star_genomegenerate.index // channel: [meta, STARIndex/]
+    star_index        = ch_star_genomegenerate.index // channel: [meta, STARIndex/]
     transcript_fasta // channel: [meta, *.transcripts.fasta]
+    vcf_gz            = ch_bgziptabix.output.map { meta, out -> [meta, out] } // channel: [meta, *.vcf.gz]
+    vcf_tbi           = ch_bgziptabix.index.map { meta, idx -> [meta, idx] } // channel: [meta, *.vcf.gz.tbi]
 }
 
 /*
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    UTILITY FUNCTIONS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-*/
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+UTILITY FUNCTIONS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ */
 
 def join_by_meta_id(channel1, channel2, channel3 = null) {
     if (channel3) {

workflows/references.nf

@@ -1,5 +1,4 @@
-include { PREPARE_GENOME_DNASEQ } from '../subworkflows/local/prepare_genome_dnaseq'
-include { PREPARE_GENOME_RNASEQ } from '../subworkflows/local/prepare_genome_rnaseq'
+include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome'
 
 workflow REFERENCES {
     take:
@@ -16,49 +15,41 @@ workflow REFERENCES {
     hisat2_skip_splice_sites
 
     main:
-    // Create references for rnaseq based pipelines such as nf-core/riboseq, nf-core/rnaseq, nf-core/rnavar
-    ch_prepare_genome_rnaseq = PREPARE_GENOME_RNASEQ(
+    ch_prepare_genome = PREPARE_GENOME(
         fasta,
         fasta_fai,
         gff,
         gtf,
         splice_sites,
         transcript_fasta,
-        tools,
-        hisat2_build_memory,
-        hisat2_skip_splice_sites,
-    )
-
-    // Create references for dnaseq based pipelines such as nf-core/sarek
-    ch_prepare_genome_dnaseq = PREPARE_GENOME_DNASEQ(
-        fasta,
-        fasta_fai.mix(ch_prepare_genome_rnaseq.fasta_fai).unique(),
         vcf,
         altliftoverfile,
         tools,
+        hisat2_build_memory,
+        hisat2_skip_splice_sites,
     )
 
     emit:
-    bowtie1_index     = ch_prepare_genome_rnaseq.bowtie1_index
-    bowtie2_index     = ch_prepare_genome_rnaseq.bowtie2_index
-    bwamem1_index     = ch_prepare_genome_dnaseq.bwamem1_index
-    bwamem2_index     = ch_prepare_genome_dnaseq.bwamem2_index
-    dragmap_hashmap   = ch_prepare_genome_dnaseq.dragmap_hashmap
+    bowtie1_index     = ch_prepare_genome.bowtie1_index
+    bowtie2_index     = ch_prepare_genome.bowtie2_index
+    bwamem1_index     = ch_prepare_genome.bwamem1_index
+    bwamem2_index     = ch_prepare_genome.bwamem2_index
+    dragmap_hashmap   = ch_prepare_genome.dragmap_hashmap
     fasta
-    fasta_dict        = ch_prepare_genome_dnaseq.fasta_dict
-    fasta_fai         = ch_prepare_genome_dnaseq.fasta_fai
-    fasta_sizes       = ch_prepare_genome_rnaseq.fasta_sizes
+    fasta_dict        = ch_prepare_genome.fasta_dict
+    fasta_fai         = ch_prepare_genome.fasta_fai
+    fasta_sizes       = ch_prepare_genome.fasta_sizes
     gff
-    gtf               = ch_prepare_genome_rnaseq.gtf
-    hisat2_index      = ch_prepare_genome_rnaseq.hisat2_index
-    intervals_bed     = ch_prepare_genome_dnaseq.intervals_bed
-    kallisto_index    = ch_prepare_genome_rnaseq.kallisto_index
-    msisensorpro_list = ch_prepare_genome_dnaseq.msisensorpro_list
-    rsem_index        = ch_prepare_genome_rnaseq.rsem_index
-    salmon_index      = ch_prepare_genome_rnaseq.salmon_index
-    snapaligner_index = ch_prepare_genome_dnaseq.snapaligner_index
-    splice_sites      = ch_prepare_genome_rnaseq.splice_sites
-    star_index        = ch_prepare_genome_rnaseq.star_index
-    transcript_fasta  = ch_prepare_genome_rnaseq.transcript_fasta
-    vcf_tbi           = ch_prepare_genome_dnaseq.vcf_tbi
+    gtf               = ch_prepare_genome.gtf
+    hisat2_index      = ch_prepare_genome.hisat2_index
+    intervals_bed     = ch_prepare_genome.intervals_bed
+    kallisto_index    = ch_prepare_genome.kallisto_index
+    msisensorpro_list = ch_prepare_genome.msisensorpro_list
+    rsem_index        = ch_prepare_genome.rsem_index
+    salmon_index      = ch_prepare_genome.salmon_index
+    snapaligner_index = ch_prepare_genome.snapaligner_index
+    splice_sites      = ch_prepare_genome.splice_sites
+    star_index        = ch_prepare_genome.star_index
+    transcript_fasta  = ch_prepare_genome.transcript_fasta
+    vcf_tbi           = ch_prepare_genome.vcf_tbi
 }