Skip to content

Replace .out channel access with typed workflow return assignment #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
maxulysse merged 1 commit into from
Sep 24, 2026
Merged

Replace .out channel access with typed workflow return assignment #157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- [#151](https://github.com/nf-core/references/pull/151) - Enable static typing and replace `collectFile` with `WRITE_FILE`
- [#153](https://github.com/nf-core/references/pull/153) - Update modules
- [#156](https://github.com/nf-core/references/pull/156) - Replace legacy `groupTuple` and `transpose` operators with `groupBy` and `flatMap`
- [#157](https://github.com/nf-core/references/pull/157) - Replace `.out` channel access with typed workflow return assignment

### Removed

Expand Down
166 changes: 83 additions & 83 deletions main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,19 @@ workflow NFCORE_REFERENCES {

main:

DATASHEET_TO_CHANNEL(references, tools)
ch_datasheet = DATASHEET_TO_CHANNEL(references, tools)

// References that need to be extracted
// (VCFs are not extracted)
ascat_alleles_input = need_extract(DATASHEET_TO_CHANNEL.out.ascat_alleles, 'ascat_alleles')
ascat_loci_input = need_extract(DATASHEET_TO_CHANNEL.out.ascat_loci, 'ascat_loci')
ascat_loci_gc_input = need_extract(DATASHEET_TO_CHANNEL.out.ascat_loci_gc, 'ascat_loci_gc')
ascat_loci_rt_input = need_extract(DATASHEET_TO_CHANNEL.out.ascat_loci_rt, 'ascat_loci_rt')
chr_dir_input = need_extract(DATASHEET_TO_CHANNEL.out.chr_dir, 'chr_dir')
ascat_alleles_input = need_extract(ch_datasheet.ascat_alleles, 'ascat_alleles')
ascat_loci_input = need_extract(ch_datasheet.ascat_loci, 'ascat_loci')
ascat_loci_gc_input = need_extract(ch_datasheet.ascat_loci_gc, 'ascat_loci_gc')
ascat_loci_rt_input = need_extract(ch_datasheet.ascat_loci_rt, 'ascat_loci_rt')
chr_dir_input = need_extract(ch_datasheet.chr_dir, 'chr_dir')

fasta_download_input = need_ncbi_download(DATASHEET_TO_CHANNEL.out.fasta, 'fasta')
gff_download_input = need_ncbi_download(DATASHEET_TO_CHANNEL.out.gff, 'gff')
gtf_download_input = need_ncbi_download(DATASHEET_TO_CHANNEL.out.gtf, 'gtf')
fasta_download_input = need_ncbi_download(ch_datasheet.fasta, 'fasta')
gff_download_input = need_ncbi_download(ch_datasheet.gff, 'gff')
gtf_download_input = need_ncbi_download(ch_datasheet.gtf, 'gtf')

ncbi_download_input = fasta_download_input.to_download
.mix(gff_download_input.to_download, gtf_download_input.to_download)
Expand All @@ -80,7 +80,7 @@ workflow NFCORE_REFERENCES {
[merged_meta, includes]
}

NCBIDATASETSCLI_DATASETS(ncbi_download_input)
ch_ncbidatasetscli_datasets = NCBIDATASETSCLI_DATASETS(ncbi_download_input)

fasta_input = need_extract(fasta_download_input.not_downloaded, 'fasta')
gff_input = need_extract(gff_download_input.not_downloaded, 'gff')
Expand All @@ -100,69 +100,69 @@ workflow NFCORE_REFERENCES {
)

// Extract references from any archive format
ARCHIVE_EXTRACT(archive_to_extract)
ch_archive_extract = ARCHIVE_EXTRACT(archive_to_extract)

// return to the appropriate channels
extracted_fasta = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'fasta' }
extracted_gff = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'gff' }
extracted_gtf = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'gtf' }
extracted_fasta = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'fasta' }
extracted_gff = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'gff' }
extracted_gtf = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'gtf' }

// This is a confidence check
def assigned_references = ['ascat_alleles', 'ascat_loci', 'ascat_loci_gc', 'ascat_loci_rt', 'chr_dir', 'fasta', 'gff', 'gtf'] as Set
ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> !(meta.reference in assigned_references) }.view { reference -> log.warn("Non assigned extracted reference: " + reference) }
ch_archive_extract.extracted.filter { meta, _ref -> !(meta.reference in assigned_references) }.view { reference -> log.warn("Non assigned extracted reference: " + reference) }

// WORKFLOW: Run pipeline
// Mix the references that were extracted with the references that did not need to be extracted
// Some references are not extracted because they are usually not stored in an archived format
// TODO: check if more references need to be extracted
altliftoverfile = false

fasta = fasta_input.not_extracted.mix(extracted_fasta, NCBIDATASETSCLI_DATASETS.out.fna.map { meta, file -> [meta + record(reference: 'fasta', file: 'fasta'), file] })
gff = gff_input.not_extracted.mix(extracted_gff, NCBIDATASETSCLI_DATASETS.out.gff.map { meta, file -> [meta + record(reference: 'gff', file: 'gff'), file] })
gtf = gtf_input.not_extracted.mix(extracted_gtf, NCBIDATASETSCLI_DATASETS.out.gtf.map { meta, file -> [meta + record(reference: 'gtf', file: 'gtf'), file] })
fasta = fasta_input.not_extracted.mix(extracted_fasta, ch_ncbidatasetscli_datasets.fna.map { meta, file -> [meta + record(reference: 'fasta', file: 'fasta'), file] })
gff = gff_input.not_extracted.mix(extracted_gff, ch_ncbidatasetscli_datasets.gff.map { meta, file -> [meta + record(reference: 'gff', file: 'gff'), file] })
gtf = gtf_input.not_extracted.mix(extracted_gtf, ch_ncbidatasetscli_datasets.gtf.map { meta, file -> [meta + record(reference: 'gtf', file: 'gtf'), file] })

REFERENCES(
ch_references = REFERENCES(
altliftoverfile,
fasta,
DATASHEET_TO_CHANNEL.out.fasta_fai,
ch_datasheet.fasta_fai,
gff,
gtf,
DATASHEET_TO_CHANNEL.out.splice_sites,
DATASHEET_TO_CHANNEL.out.transcript_fasta,
DATASHEET_TO_CHANNEL.out.vcf,
ch_datasheet.splice_sites,
ch_datasheet.transcript_fasta,
ch_datasheet.vcf,
tools,
params.hisat2_build_memory,
params.hisat2_skip_splice_sites,
)

emit:
ascat_alleles = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'ascat_alleles' }
ascat_loci = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci' }
ascat_loci_gc = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci_gc' }
ascat_loci_rt = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci_rt' }
bowtie1_index = REFERENCES.out.bowtie1_index
bowtie2_index = REFERENCES.out.bowtie2_index
bwamem1_index = REFERENCES.out.bwamem1_index
bwamem2_index = REFERENCES.out.bwamem2_index
chr_dir = ARCHIVE_EXTRACT.out.extracted.filter { meta, _ref -> meta.reference == 'chr_dir' }
dragmap_hashmap = REFERENCES.out.dragmap_hashmap
fasta = extracted_fasta.mix(NCBIDATASETSCLI_DATASETS.out.fna.map { meta, file -> [meta + record(reference: 'fasta', file: 'fasta'), file] })
fasta_dict = REFERENCES.out.fasta_dict
fasta_fai = REFERENCES.out.fasta_fai
fasta_sizes = REFERENCES.out.fasta_sizes
gff = extracted_gff.mix(NCBIDATASETSCLI_DATASETS.out.gff.map { meta, file -> [meta + record(reference: 'gff', file: 'gff'), file] })
gtf = REFERENCES.out.gtf
hisat2_index = REFERENCES.out.hisat2_index
intervals_bed = REFERENCES.out.intervals_bed
kallisto_index = REFERENCES.out.kallisto_index
msisensorpro_list = REFERENCES.out.msisensorpro_list
rsem_index = REFERENCES.out.rsem_index
salmon_index = REFERENCES.out.salmon_index
snapaligner_index = REFERENCES.out.snapaligner_index
splice_sites = REFERENCES.out.splice_sites
star_index = REFERENCES.out.star_index
transcript_fasta = REFERENCES.out.transcript_fasta
vcf_tbi = REFERENCES.out.vcf_tbi
ascat_alleles = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'ascat_alleles' }
ascat_loci = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci' }
ascat_loci_gc = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci_gc' }
ascat_loci_rt = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'ascat_loci_rt' }
bowtie1_index = ch_references.bowtie1_index
bowtie2_index = ch_references.bowtie2_index
bwamem1_index = ch_references.bwamem1_index
bwamem2_index = ch_references.bwamem2_index
chr_dir = ch_archive_extract.extracted.filter { meta, _ref -> meta.reference == 'chr_dir' }
dragmap_hashmap = ch_references.dragmap_hashmap
fasta = extracted_fasta.mix(ch_ncbidatasetscli_datasets.fna.map { meta, file -> [meta + record(reference: 'fasta', file: 'fasta'), file] })
fasta_dict = ch_references.fasta_dict
fasta_fai = ch_references.fasta_fai
fasta_sizes = ch_references.fasta_sizes
gff = extracted_gff.mix(ch_ncbidatasetscli_datasets.gff.map { meta, file -> [meta + record(reference: 'gff', file: 'gff'), file] })
gtf = ch_references.gtf
hisat2_index = ch_references.hisat2_index
intervals_bed = ch_references.intervals_bed
kallisto_index = ch_references.kallisto_index
msisensorpro_list = ch_references.msisensorpro_list
rsem_index = ch_references.rsem_index
salmon_index = ch_references.salmon_index
snapaligner_index = ch_references.snapaligner_index
splice_sites = ch_references.splice_sites
star_index = ch_references.star_index
transcript_fasta = ch_references.transcript_fasta
vcf_tbi = ch_references.vcf_tbi
}

/*
Expand Down Expand Up @@ -285,7 +285,7 @@ workflow {
def tools = defineToolsList(params.tools_bundle, params.tools, params.skip_tools)

// SUBWORKFLOW: Run initialisation tasks
PIPELINE_INITIALISATION(
ch_pipeline_initialisation = PIPELINE_INITIALISATION(
params.version,
params.validate_params,
params.monochrome_logs,
Expand All @@ -301,7 +301,7 @@ workflow {
)

// WORKFLOW: Run main workflow
NFCORE_REFERENCES(PIPELINE_INITIALISATION.out.references, tools)
ch_nfcore_references = NFCORE_REFERENCES(ch_pipeline_initialisation.references, tools)

// VERSIONS
collated_versions = softwareVersionsToYAML(channel.topic('versions'))
Expand Down Expand Up @@ -330,7 +330,7 @@ workflow {

multiqc_files = multiqc_files.mix(ch_collected_files)

MULTIQC(
ch_multiqc = MULTIQC(
multiqc_files.flatten().collect().map { files ->
[
[id: 'references'],
Expand All @@ -344,7 +344,7 @@ workflow {
]
}
)
multiqc_report = MULTIQC.out.report.map { _meta, report -> [report] }.toList()
multiqc_report = ch_multiqc.report.map { _meta, report -> [report] }.toList()

// SUBWORKFLOW: Run completion tasks
PIPELINE_COMPLETION(
Expand All @@ -353,39 +353,39 @@ workflow {
params.plaintext_email,
params.outdir,
params.monochrome_logs,
MULTIQC.out.report.toList(),
ch_multiqc.report.toList(),
)

publish:
multiqc = MULTIQC.out.data.mix(MULTIQC.out.plots, MULTIQC.out.report)
multiqc = ch_multiqc.data.mix(ch_multiqc.plots, ch_multiqc.report)
versions = ch_collected_files.filter { file -> file.name.contains('versions') }
ascat_alleles = NFCORE_REFERENCES.out.ascat_alleles
ascat_loci = NFCORE_REFERENCES.out.ascat_loci
ascat_loci_gc = NFCORE_REFERENCES.out.ascat_loci_gc
ascat_loci_rt = NFCORE_REFERENCES.out.ascat_loci_rt
bowtie1_index = NFCORE_REFERENCES.out.bowtie1_index
bowtie2_index = NFCORE_REFERENCES.out.bowtie2_index
bwamem1_index = NFCORE_REFERENCES.out.bwamem1_index
bwamem2_index = NFCORE_REFERENCES.out.bwamem2_index
chr_dir = NFCORE_REFERENCES.out.chr_dir
dragmap_hashmap = NFCORE_REFERENCES.out.dragmap_hashmap
fasta = NFCORE_REFERENCES.out.fasta
fasta_dict = NFCORE_REFERENCES.out.fasta_dict
fasta_fai = NFCORE_REFERENCES.out.fasta_fai
fasta_sizes = NFCORE_REFERENCES.out.fasta_sizes
gff = NFCORE_REFERENCES.out.gff
gtf = NFCORE_REFERENCES.out.gtf
hisat2_index = NFCORE_REFERENCES.out.hisat2_index
intervals_bed = NFCORE_REFERENCES.out.intervals_bed
kallisto_index = NFCORE_REFERENCES.out.kallisto_index
msisensorpro_list = NFCORE_REFERENCES.out.msisensorpro_list
rsem_index = NFCORE_REFERENCES.out.rsem_index
salmon_index = NFCORE_REFERENCES.out.salmon_index
snapaligner_index = NFCORE_REFERENCES.out.snapaligner_index
splice_sites = NFCORE_REFERENCES.out.splice_sites
star_index = NFCORE_REFERENCES.out.star_index
transcript_fasta = NFCORE_REFERENCES.out.transcript_fasta
vcf_tbi = NFCORE_REFERENCES.out.vcf_tbi
ascat_alleles = ch_nfcore_references.ascat_alleles
ascat_loci = ch_nfcore_references.ascat_loci
ascat_loci_gc = ch_nfcore_references.ascat_loci_gc
ascat_loci_rt = ch_nfcore_references.ascat_loci_rt
bowtie1_index = ch_nfcore_references.bowtie1_index
bowtie2_index = ch_nfcore_references.bowtie2_index
bwamem1_index = ch_nfcore_references.bwamem1_index
bwamem2_index = ch_nfcore_references.bwamem2_index
chr_dir = ch_nfcore_references.chr_dir
dragmap_hashmap = ch_nfcore_references.dragmap_hashmap
fasta = ch_nfcore_references.fasta
fasta_dict = ch_nfcore_references.fasta_dict
fasta_fai = ch_nfcore_references.fasta_fai
fasta_sizes = ch_nfcore_references.fasta_sizes
gff = ch_nfcore_references.gff
gtf = ch_nfcore_references.gtf
hisat2_index = ch_nfcore_references.hisat2_index
intervals_bed = ch_nfcore_references.intervals_bed
kallisto_index = ch_nfcore_references.kallisto_index
msisensorpro_list = ch_nfcore_references.msisensorpro_list
rsem_index = ch_nfcore_references.rsem_index
salmon_index = ch_nfcore_references.salmon_index
snapaligner_index = ch_nfcore_references.snapaligner_index
splice_sites = ch_nfcore_references.splice_sites
star_index = ch_nfcore_references.star_index
transcript_fasta = ch_nfcore_references.transcript_fasta
vcf_tbi = ch_nfcore_references.vcf_tbi
}

output {
Expand Down
38 changes: 19 additions & 19 deletions subworkflows/local/prepare_genome_dnaseq/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,26 @@ workflow PREPARE_GENOME_DNASEQ {

run_faidx = ('faidx' in tools) && !('intervals' in tools || 'sizes' in tools)

BWAMEM1_INDEX(fasta.filter { meta, _fasta -> 'bwamem1' in tools && meta.run_bwamem1 })
ch_bwamem1_index = BWAMEM1_INDEX(fasta.filter { meta, _fasta -> 'bwamem1' in tools && meta.run_bwamem1 })

BWAMEM2_INDEX(fasta.filter { meta, _fasta -> 'bwamem2' in tools && meta.run_bwamem2 })
ch_bwamem2_index = BWAMEM2_INDEX(fasta.filter { meta, _fasta -> 'bwamem2' in tools && meta.run_bwamem2 })

DRAGMAP_HASHTABLE(fasta.filter { meta, _fasta -> 'dragmap' in tools && meta.run_dragmap })
ch_dragmap_hashtable = DRAGMAP_HASHTABLE(fasta.filter { meta, _fasta -> 'dragmap' in tools && meta.run_dragmap })

GATK4_CREATESEQUENCEDICTIONARY(fasta.filter { meta, _fasta -> 'createsequencedictionary' in tools && meta.run_createsequencedictionary })
ch_gatk4_dict = GATK4_CREATESEQUENCEDICTIONARY(fasta.filter { meta, _fasta -> 'createsequencedictionary' in tools && meta.run_createsequencedictionary })

// Do not generate sizes for DNAseq
generate_sizes = false

SAMTOOLS_FAIDX(fasta.filter { meta, _fasta -> run_faidx && meta.run_faidx }.map { meta, fasta_ -> [meta, fasta_, []] }, generate_sizes)
ch_samtools_faidx = SAMTOOLS_FAIDX(fasta.filter { meta, _fasta -> run_faidx && meta.run_faidx }.map { meta, fasta_ -> [meta, fasta_, []] }, generate_sizes)

fasta_fai = fasta_fai.mix(SAMTOOLS_FAIDX.out.fai)
fasta_fai = fasta_fai.mix(ch_samtools_faidx.fai)

BUILD_INTERVALS(fasta_fai.filter { meta, _fasta_fai -> 'intervals' in tools && meta.run_intervals }, [], false)
ch_build_intervals = BUILD_INTERVALS(fasta_fai.filter { meta, _fasta_fai -> 'intervals' in tools && meta.run_intervals }, [], false)

MSISENSORPRO_SCAN(fasta.filter { meta, _fasta -> 'msisensorpro' in tools && meta.run_msisensorpro })
ch_msisensorpro_scan = MSISENSORPRO_SCAN(fasta.filter { meta, _fasta -> 'msisensorpro' in tools && meta.run_msisensorpro })

HTSLIB_BGZIPTABIX(
ch_bgziptabix = HTSLIB_BGZIPTABIX(
vcf.filter { meta, _vcf -> 'tabix' in tools && meta.run_tabix }.map { meta, vcf_ -> [meta, vcf_, null, null] },
"compress",
true,
Expand All @@ -50,17 +50,17 @@ workflow PREPARE_GENOME_DNASEQ {
? fasta.combine(altliftoverfile).map { meta, fasta_, altliftoverfile_ -> [meta, fasta_, [], [], altliftoverfile_] }
: fasta.map { meta, fasta_ -> [meta, fasta_, [], [], []] }

SNAPALIGNER_INDEX(fasta_for_snapaligner.filter { meta, _fasta, _altcontigfile, _nonaltcontigfile, _altliftoverfile -> 'snapaligner' in tools && meta.run_snapaligner })
ch_snapaligner_index = SNAPALIGNER_INDEX(fasta_for_snapaligner.filter { meta, _fasta, _altcontigfile, _nonaltcontigfile, _altliftoverfile -> 'snapaligner' in tools && meta.run_snapaligner })

emit:
bwamem1_index = BWAMEM1_INDEX.out.index // channel: [meta, BWAmemIndex/]
bwamem2_index = BWAMEM2_INDEX.out.index // channel: [meta, BWAmem2memIndex/]
dragmap_hashmap = DRAGMAP_HASHTABLE.out.hashmap // channel: [meta, DragmapHashtable/]
fasta_dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict // channel: [meta, *.fa(sta).dict]
bwamem1_index = ch_bwamem1_index.index // channel: [meta, BWAmemIndex/]
bwamem2_index = ch_bwamem2_index.index // channel: [meta, BWAmem2memIndex/]
dragmap_hashmap = ch_dragmap_hashtable.hashmap // channel: [meta, DragmapHashtable/]
fasta_dict = ch_gatk4_dict.dict // channel: [meta, *.fa(sta).dict]
fasta_fai // channel: [meta, *.fa(sta).fai]
intervals_bed = BUILD_INTERVALS.out.output // channel: [meta, *.bed]
msisensorpro_list = MSISENSORPRO_SCAN.out.list // channel: [meta, *.list]
snapaligner_index = SNAPALIGNER_INDEX.out.index // channel: [meta, snap/]
vcf_gz = HTSLIB_BGZIPTABIX.out.output.map { meta, out -> [meta, out] } // channel: [meta, *.vcf.gz]
vcf_tbi = HTSLIB_BGZIPTABIX.out.index.map { meta, idx -> [meta, idx] } // channel: [meta, *.vcf.gz.tbi]
intervals_bed = ch_build_intervals.output // channel: [meta, *.bed]
msisensorpro_list = ch_msisensorpro_scan.list // channel: [meta, *.list]
snapaligner_index = ch_snapaligner_index.index // channel: [meta, snap/]
vcf_gz = ch_bgziptabix.output.map { meta, out -> [meta, out] } // channel: [meta, *.vcf.gz]
vcf_tbi = ch_bgziptabix.index.map { meta, idx -> [meta, idx] } // channel: [meta, *.vcf.gz.tbi]
}
Loading
Loading