From 63640bf1b9032f2973753534208dbb5e08da0859 Mon Sep 17 00:00:00 2001 From: Vkale1 Date: Thu, 15 Oct 2026 16:21:35 +0200 Subject: [PATCH 1/6] first draft for bam file as raw read input, uses draft containers and bam tests will currently fail until toolkit containers are updated --- assets/schema_input_reads.json | 81 +++++++++++++++++-- conf/test_reads_long_bam.config | 35 ++++++++ modules/local/create_reads_manifest/main.nf | 11 +-- modules/local/create_reads_manifest/meta.yml | 14 ++-- .../create_reads_manifest/tests/main.nf.test | 44 ++++++++++ nextflow.config | 1 + tests/reads_long_bam.nf.test | 45 +++++++++++ workflows/readsubmit.nf | 18 +++-- 8 files changed, 223 insertions(+), 26 deletions(-) create mode 100644 conf/test_reads_long_bam.config create mode 100644 tests/reads_long_bam.nf.test diff --git a/assets/schema_input_reads.json b/assets/schema_input_reads.json index c511c3cc..963fe107 100644 --- a/assets/schema_input_reads.json +++ b/assets/schema_input_reads.json @@ -22,12 +22,21 @@ "meta": ["sample_accession"] }, "fastq_1": { - "type": "string", - "format": "file-path", - "exists": true, - "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$", + "anyOf": [ + { + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped)" + "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Leave empty if reads are provided as BAM" }, "fastq_2": { "anyOf": [ @@ -46,6 +55,23 @@ "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads" }, + "bam": { + "anyOf": [ + { + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.bam$" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "errorMessage": "BAM file must have extension '.bam'", + "description": "Path to reads in unaligned or aligned BAM format. Use instead of fastq_1/fastq_2" + }, "platform": { "type": "string", "enum": [ @@ -223,12 +249,53 @@ "required": [ "id", "sample_accession", - "fastq_1", "platform", "instrument", "library_source", "library_selection", "library_strategy" - ] + ], + "oneOf": [ + { + "properties": { + "fastq_1": { + "type": "string", + "minLength": 1 + } + }, + "required": ["fastq_1"] + }, + { + "properties": { + "bam": { + "type": "string", + "minLength": 1 + } + }, + "required": ["bam"] + } + ], + "if": { + "properties": { + "bam": { + "type": "string", + "minLength": 1 + } + }, + "required": ["bam"] + }, + "then": { + "properties": { + "fastq_1": { + "maxLength": 0, + "errorMessage": "fastq_1 must be empty when reads are provided as BAM" + }, + "fastq_2": { + "maxLength": 0, + "errorMessage": "fastq_2 must be empty when reads are provided as BAM" + } + } + }, + "errorMessage": "Reads must be provided either as FASTQ (fastq_1, optionally fastq_2) or as a single BAM (bam, with fastq_1 and fastq_2 empty)" } } diff --git a/conf/test_reads_long_bam.config b/conf/test_reads_long_bam.config new file mode 100644 index 00000000..8b47e3da --- /dev/null +++ b/conf/test_reads_long_bam.config @@ -0,0 +1,35 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_reads_long_bam, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode reads long-read BAM profile' + config_profile_description = 'Minimal test profile for long-read BAM reads submission' + + // Input data + // TODO: move to params.pipelines_testdata_base_path once the BAM + samplesheet are uploaded to nf-core/test-datasets + input = "${projectDir}/tests/test_data/samplesheets/samplesheet_reads_bam.csv" + outdir = 'test_output' + + mode = "reads" + submission_study = "PRJEB98843" + centre_name = "TEST_CENTER" + + test_upload = true +} diff --git a/modules/local/create_reads_manifest/main.nf b/modules/local/create_reads_manifest/main.nf index ded09e79..cfea512a 100644 --- a/modules/local/create_reads_manifest/main.nf +++ b/modules/local/create_reads_manifest/main.nf @@ -5,10 +5,10 @@ process CREATE_READS_MANIFEST { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1': - 'biocontainers/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1' }" + 'quay.io/microbiome-informatics/mgnify-pipelines-toolkit:feat-raw-reads' }" input: - tuple val(meta), path(fastq_files) + tuple val(meta), path(read_files) val(study_accession) output: @@ -21,8 +21,9 @@ process CREATE_READS_MANIFEST { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def fastq_list = fastq_files instanceof List ? fastq_files : [fastq_files] - def fastq_args = fastq_list.collect { "--fastq ${it.name}" }.join(' \\\n ') + def read_list = read_files instanceof List ? read_files : [read_files] + def read_flag = meta.file_type == 'bam' ? '--bam' : '--fastq' + def read_args = read_list.collect { file -> "${read_flag} ${file.name}" }.join(' \\\n ') def opt_insert = meta.insert_size ? "--insert-size ${meta.insert_size}" : '' def opt_lib_name = meta.library_name ? "--library-name '${meta.library_name}'" : '' def opt_desc = meta.description ? "--description '${meta.description}'" : '' @@ -38,7 +39,7 @@ process CREATE_READS_MANIFEST { --library-selection '${meta.library_selection}' \\ --library-strategy '${meta.library_strategy}' \\ --output ${prefix}.manifest \\ - ${fastq_args} \\ + ${read_args} \\ ${opt_insert} \\ ${opt_lib_name} \\ ${opt_desc} \\ diff --git a/modules/local/create_reads_manifest/meta.yml b/modules/local/create_reads_manifest/meta.yml index b8bead49..b95ff690 100644 --- a/modules/local/create_reads_manifest/meta.yml +++ b/modules/local/create_reads_manifest/meta.yml @@ -29,13 +29,15 @@ input: description: | Groovy Map containing sample information, including platform, instrument, library_source, library_selection, library_strategy, - sample_accession, and optionally insert_size, library_name, - description. - e.g. `[ id:'sample1', platform:'ILLUMINA', instrument:'Illumina HiSeq 2500' ]` - - fastq_files: + sample_accession, file_type ('fastq' or 'bam'), and optionally + insert_size, library_name, description. + e.g. `[ id:'sample1', platform:'ILLUMINA', instrument:'Illumina HiSeq 2500', file_type:'fastq' ]` + - read_files: type: file - description: One or two FASTQ files (single-end or paired-end reads). - pattern: "*.{fastq,fastq.gz,fq,fq.gz}" + description: | + One or two FASTQ files (single-end or paired-end reads), or a single + BAM file. Written to the manifest as FASTQ or BAM according to meta.file_type. + pattern: "*.{fastq,fastq.gz,fq,fq.gz,bam}" - - study_accession: type: value description: ENA study accession to submit reads to. diff --git a/modules/local/create_reads_manifest/tests/main.nf.test b/modules/local/create_reads_manifest/tests/main.nf.test index 3c33494c..68151f70 100644 --- a/modules/local/create_reads_manifest/tests/main.nf.test +++ b/modules/local/create_reads_manifest/tests/main.nf.test @@ -96,4 +96,48 @@ nextflow_process { } } + test("CREATE_READS_MANIFEST - long-read BAM") { + when { + process { + """ + input[0] = [ + [ + id: 'test_bam', + sample_accession: 'SAMEA12345', + platform: 'OXFORD_NANOPORE', + instrument: 'MinION', + library_source: 'METAGENOMIC', + library_selection: 'RANDOM', + library_strategy: 'WGS', + insert_size: null, + library_name: null, + description: null, + file_type: 'bam' + ], + // TODO: switch to params.pipelines_testdata_base_path once uploaded to nf-core/test-datasets + file("${projectDir}/tests/test_data/reads/tiny_ont.bam", checkIfExists: true) + ] + input[1] = "PRJEB12345" + """ + } + } + + then { + assert process.success + assertAll( + { assert process.out.manifest.size() == 1 }, + { + def manifestContent = path(process.out.manifest[0][1]).text + def lines = manifestContent.readLines() + // 8 required fields + 1 BAM entry = 9 lines + assert lines.size() == 9 + assert manifestContent.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.count { it.startsWith("BAM\t") } == 1 + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } + ) + } + } + } diff --git a/nextflow.config b/nextflow.config index d3911f83..f9769b11 100644 --- a/nextflow.config +++ b/nextflow.config @@ -255,6 +255,7 @@ profiles { test_assembly_no_coverage_paired_reads { includeConfig 'conf/test_assembly_no_coverage_paired_reads.config' } test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' } test_reads_paired { includeConfig 'conf/test_reads_paired.config' } + test_reads_long_bam { includeConfig 'conf/test_reads_long_bam.config' } } // Load nf-core custom profiles from different institutions diff --git a/tests/reads_long_bam.nf.test b/tests/reads_long_bam.nf.test new file mode 100644 index 00000000..7df84dc9 --- /dev/null +++ b/tests/reads_long_bam.nf.test @@ -0,0 +1,45 @@ +nextflow_pipeline { + + name "Test reads submission workflow stub - long-read BAM" + script "../main.nf" + tag "pipeline" + tag "mode_reads" + tag "test_reads_long_bam" + profile "test_reads_long_bam" + + test("-profile test_reads_long_bam") { + + when { + params { + outdir = "$outputDir" + // TODO: copying BAM, drop once the BAM + samplesheet are hosted in nf-core/test-datasets + def samplesheet = new File("${launchDir}/samplesheet_reads_bam.csv") + samplesheet.parentFile.mkdirs() + samplesheet.text = new File("${projectDir}/tests/test_data/samplesheets/samplesheet_reads_bam.csv").text + .replace("tests/test_data/reads/tiny_ont.bam", "${projectDir}/tests/test_data/reads/tiny_ont.bam") + input = samplesheet.toString() + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() } + ) + } + } +} diff --git a/workflows/readsubmit.nf b/workflows/readsubmit.nf index 8bdff882..e2a3fa30 100644 --- a/workflows/readsubmit.nf +++ b/workflows/readsubmit.nf @@ -45,14 +45,16 @@ workflow READSUBMIT { // --------- Create reads channel with proper metadata structure reads_ch = ch_samplesheet - .map { meta, reads_1, reads_2 -> - def new_meta = meta + [single_end: !reads_2] - if ( !new_meta.single_end ) { + .map { meta, reads_1, reads_2, bam -> + if ( bam ) { + // Reads provided as BAM + [meta + [single_end: true, file_type: 'bam'], bam] + } else if ( reads_2 ) { // If paired end reads - [new_meta, [reads_1, reads_2]] + [meta + [single_end: false, file_type: 'fastq'], [reads_1, reads_2]] } else { // If single end - [new_meta, reads_1] + [meta + [single_end: true, file_type: 'fastq'], reads_1] } } @@ -82,10 +84,10 @@ workflow READSUBMIT { study_accession_ch.first() ) - // Prepare input for submission with manifest and fastq files + // Prepare input for submission with manifest and read files submission_input = reads_ch.join(CREATE_READS_MANIFEST.out.manifest) - .map { meta, fastq, manifest -> - [meta, fastq, manifest] + .map { meta, reads, manifest -> + [meta, reads, manifest] } // --------- Upload data to ENA From 16962f73034ea367d16574c31ad91fa84d8c3ee7 Mon Sep 17 00:00:00 2001 From: Vkale1 Date: Fri, 16 Oct 2026 16:00:03 +0200 Subject: [PATCH 2/6] change test datasets path, symlink read files instead of copy, update docs --- conf/modules.config | 6 +++ conf/test_reads_long_bam.config | 3 +- docs/usage.md | 39 ++++++++++--------- .../create_reads_manifest/tests/main.nf.test | 4 +- tests/reads_long_bam.nf.test | 8 +--- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index ce7df3f9..86454291 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -176,6 +176,12 @@ process { ] } + // Reads can be multi or 100s of GB (e.g. long-read BAMs): symlink instead of copying them before upload. + // assemblies, bins and mags modes keep the default stageInMode 'copy'. + withName: '.*:READSUBMIT:SUBMIT' { + stageInMode = 'symlink' + } + withName: 'REGISTERSTUDY|GENERATE_ASSEMBLY_MANIFEST|CREATE_READS_MANIFEST' { publishDir = [ enabled: false diff --git a/conf/test_reads_long_bam.config b/conf/test_reads_long_bam.config index 8b47e3da..14d70f97 100644 --- a/conf/test_reads_long_bam.config +++ b/conf/test_reads_long_bam.config @@ -23,8 +23,7 @@ params { config_profile_description = 'Minimal test profile for long-read BAM reads submission' // Input data - // TODO: move to params.pipelines_testdata_base_path once the BAM + samplesheet are uploaded to nf-core/test-datasets - input = "${projectDir}/tests/test_data/samplesheets/samplesheet_reads_bam.csv" + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' outdir = 'test_output' mode = "reads" diff --git a/docs/usage.md b/docs/usage.md index 14c38e20..a780078b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -156,25 +156,27 @@ You will need to create a samplesheet with information about the raw reads you w Example: ```csv title="samplesheet_reads.csv" -id,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description -illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ -pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing +id,sample_accession,fastq_1,fastq_2,bam,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description +illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ +pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing +nanopore_run_001,SAMEA7654322,,,data/nanopore_reads.bam,OXFORD_NANOPORE,MinION,METAGENOMIC,RANDOM,WGS,,ONT_library_003,Long-read sequencing provided as BAM ``` -| Column | Required | Description | -| ------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | -| `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | -| `fastq_1` | Yes | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). | -| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads. | -| `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | -| `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | -| `library_source` | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. ENA [documantation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source). | -| `library_selection` | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection) for complete list. | -| `library_strategy` | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy) for complete list. | -| `insert_size` | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | -| `library_name` | No | Descriptive library name (optional). | -| `description` | No | Free-text description of the experiment (optional). | +| Column | Required | Description | +| ------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | +| `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | +| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required unless `bam` is provided. | +| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads and if `bam` is provided. | +| `bam` | Conditional | Path to reads in BAM format (`.bam`). Required unless `fastq_1` is provided. | +| `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | +| `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | +| `library_source` | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. ENA [documantation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source). | +| `library_selection` | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection) for complete list. | +| `library_strategy` | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy) for complete list. | +| `insert_size` | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | +| `library_name` | No | Descriptive library name (optional). | +| `description` | No | Free-text description of the experiment (optional). | ### `reads` parameters and example @@ -197,7 +199,8 @@ nextflow run nf-core/seqsubmit \ `reads` mode does not support the following: -- **Reads in non-FASTQ formats**: Only reads provided as FASTQ (`.fastq.gz`) can be submitted. Other formats (e.g. BAM) are not supported as input. +- **Other read formats**: Reads must be provided as FASTQ (`.fastq`/`.fq`, optionally gzipped) or BAM (`.bam`). Other formats (e.g. CRAM or `.tar.gz` archives) are not supported as input. +- **Mixed formats in one row**: A row cannot combine FASTQ and BAM files. Put them in separate rows, which are submitted as separate ENA runs. ## `metagenomic_assemblies` mode diff --git a/modules/local/create_reads_manifest/tests/main.nf.test b/modules/local/create_reads_manifest/tests/main.nf.test index 68151f70..9768ee6c 100644 --- a/modules/local/create_reads_manifest/tests/main.nf.test +++ b/modules/local/create_reads_manifest/tests/main.nf.test @@ -112,10 +112,8 @@ nextflow_process { insert_size: null, library_name: null, description: null, - file_type: 'bam' ], - // TODO: switch to params.pipelines_testdata_base_path once uploaded to nf-core/test-datasets - file("${projectDir}/tests/test_data/reads/tiny_ont.bam", checkIfExists: true) + file(params.pipelines_testdata_base_path + 'test_data/reads/tiny_ont.bam', checkIfExists: true) ] input[1] = "PRJEB12345" """ diff --git a/tests/reads_long_bam.nf.test b/tests/reads_long_bam.nf.test index 7df84dc9..661bf508 100644 --- a/tests/reads_long_bam.nf.test +++ b/tests/reads_long_bam.nf.test @@ -1,6 +1,6 @@ nextflow_pipeline { - name "Test reads submission workflow stub - long-read BAM" + name "Test reads submission workflow - long-read BAM" script "../main.nf" tag "pipeline" tag "mode_reads" @@ -12,12 +12,6 @@ nextflow_pipeline { when { params { outdir = "$outputDir" - // TODO: copying BAM, drop once the BAM + samplesheet are hosted in nf-core/test-datasets - def samplesheet = new File("${launchDir}/samplesheet_reads_bam.csv") - samplesheet.parentFile.mkdirs() - samplesheet.text = new File("${projectDir}/tests/test_data/samplesheets/samplesheet_reads_bam.csv").text - .replace("tests/test_data/reads/tiny_ont.bam", "${projectDir}/tests/test_data/reads/tiny_ont.bam") - input = samplesheet.toString() } } From 3387b30e51794b6cb4a81451d3a2e22e071c8750 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Fri, 23 Oct 2026 13:31:13 +0100 Subject: [PATCH 3/6] update containers with mgnify-toolkit and corresponding tests --- CHANGELOG.md | 3 + conf/modules.config | 2 +- conf/test_reads_long_bam.config | 4 +- conf/test_reads_long_bam_validate.config | 35 ++++++++++ .../create_reads_manifest/environment.yml | 2 +- modules/local/create_reads_manifest/main.nf | 4 +- .../create_reads_manifest/tests/main.nf.test | 1 + .../ena_webin_cli_wrapper/environment.yml | 2 +- modules/local/ena_webin_cli_wrapper/main.nf | 6 +- modules/local/registerstudy/environment.yml | 2 +- modules/local/registerstudy/main.nf | 4 +- nextflow.config | 1 + tests/assembly_complete_metadata.nf.test.snap | 6 +- ...plete_metadata_existing_study.nf.test.snap | 4 +- ...mbly_no_coverage_paired_reads.nf.test.snap | 6 +- ...mbly_no_coverage_single_reads.nf.test.snap | 6 +- tests/default.nf.test.snap | 6 +- tests/mag_complete_metadata.nf.test.snap | 6 +- ...plete_metadata_existing_study.nf.test.snap | 4 +- .../mag_no_coverage_paired_reads.nf.test.snap | 6 +- .../mag_no_coverage_single_reads.nf.test.snap | 6 +- tests/mag_no_rna_presence.nf.test.snap | 55 ++++++++++------ tests/mag_no_taxonomy.nf.test.snap | 6 +- tests/reads_long_bam.nf.test | 29 ++++++++- tests/reads_long_bam.nf.test.snap | 57 ++++++++++++++++ tests/reads_long_bam_validate.nf.test | 65 +++++++++++++++++++ tests/reads_long_bam_validate.nf.test.snap | 42 ++++++++++++ tests/reads_paired_end.nf.test.snap | 8 +-- 28 files changed, 312 insertions(+), 66 deletions(-) create mode 100644 conf/test_reads_long_bam_validate.config create mode 100644 tests/reads_long_bam.nf.test.snap create mode 100644 tests/reads_long_bam_validate.nf.test create mode 100644 tests/reads_long_bam_validate.nf.test.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b9d142..20abedeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#117](https://github.com/nf-core/seqsubmit/pull/117) and [#119](https://github.com/nf-core/seqsubmit/pull/119) + - Added BAM files support as raw reads + - Added tests for BAM submission with `-submit` and `-validate` modes - [#116](https://github.com/nf-core/seqsubmit/pull/116) - All tests except the `*_existing_study` ones now register a new submission study, so they can be run with any Webin account ([#95](https://github.com/nf-core/seqsubmit/issues/95)). The `test_mag_complete_metadata_existing_study` and `test_assembly_complete_metadata_existing_study` tests submit to the existing study `PRJEB98843` and require maintainer Webin credentials - `test_upload = true` is set explicitly in all test configs diff --git a/conf/modules.config b/conf/modules.config index 4acce5e4..c14d7f2c 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -181,7 +181,7 @@ process { withName: '.*:READSUBMIT:SUBMIT' { stageInMode = 'symlink' } - + withName: 'REGISTERSTUDY|CREATE_READS_MANIFEST' { publishDir = [ enabled: false diff --git a/conf/test_reads_long_bam.config b/conf/test_reads_long_bam.config index 14d70f97..d510081d 100644 --- a/conf/test_reads_long_bam.config +++ b/conf/test_reads_long_bam.config @@ -27,7 +27,9 @@ params { outdir = 'test_output' mode = "reads" - submission_study = "PRJEB98843" + webincli_mode = "submit" + submission_study = null + study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json' centre_name = "TEST_CENTER" test_upload = true diff --git a/conf/test_reads_long_bam_validate.config b/conf/test_reads_long_bam_validate.config new file mode 100644 index 00000000..f74f20fa --- /dev/null +++ b/conf/test_reads_long_bam_validate.config @@ -0,0 +1,35 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_reads_long_bam_validate, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode reads long-read BAM profile validate only' + config_profile_description = 'Minimal test profile for long-read BAM reads submission' + + // Input data + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' + outdir = 'test_output' + + mode = "reads" + webincli_mode = "validate" + submission_study = "PRJEB98843" + centre_name = "TEST_CENTER" + + test_upload = false +} diff --git a/modules/local/create_reads_manifest/environment.yml b/modules/local/create_reads_manifest/environment.yml index 053a4fb6..21d90955 100644 --- a/modules/local/create_reads_manifest/environment.yml +++ b/modules/local/create_reads_manifest/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda dependencies: - conda-forge::python>=3.12 - - bioconda::mgnify-pipelines-toolkit=1.5.1 + - bioconda::mgnify-pipelines-toolkit=1.5.5 diff --git a/modules/local/create_reads_manifest/main.nf b/modules/local/create_reads_manifest/main.nf index cfea512a..dba90d3a 100644 --- a/modules/local/create_reads_manifest/main.nf +++ b/modules/local/create_reads_manifest/main.nf @@ -4,8 +4,8 @@ process CREATE_READS_MANIFEST { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1': - 'quay.io/microbiome-informatics/mgnify-pipelines-toolkit:feat-raw-reads' }" + 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0 ': + 'biocontainers/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0' }" input: tuple val(meta), path(read_files) diff --git a/modules/local/create_reads_manifest/tests/main.nf.test b/modules/local/create_reads_manifest/tests/main.nf.test index 9768ee6c..56439fe7 100644 --- a/modules/local/create_reads_manifest/tests/main.nf.test +++ b/modules/local/create_reads_manifest/tests/main.nf.test @@ -112,6 +112,7 @@ nextflow_process { insert_size: null, library_name: null, description: null, + file_type: 'bam' ], file(params.pipelines_testdata_base_path + 'test_data/reads/tiny_ont.bam', checkIfExists: true) ] diff --git a/modules/local/ena_webin_cli_wrapper/environment.yml b/modules/local/ena_webin_cli_wrapper/environment.yml index fe7d46d3..93d49b8d 100644 --- a/modules/local/ena_webin_cli_wrapper/environment.yml +++ b/modules/local/ena_webin_cli_wrapper/environment.yml @@ -3,6 +3,6 @@ channels: - conda-forge - bioconda dependencies: - - "bioconda::mgnify-pipelines-toolkit=1.5.1" + - "bioconda::mgnify-pipelines-toolkit=1.5.5" - "conda-forge::openjdk" - "bioconda::ena-webin-cli=9.0.3" diff --git a/modules/local/ena_webin_cli_wrapper/main.nf b/modules/local/ena_webin_cli_wrapper/main.nf index 7a689619..2ac07077 100644 --- a/modules/local/ena_webin_cli_wrapper/main.nf +++ b/modules/local/ena_webin_cli_wrapper/main.nf @@ -8,10 +8,10 @@ process ENA_WEBIN_CLI_WRAPPER { conda "${moduleDir}/environment.yml" - // ena-webin-cli 9.0.3 + mgnify-pipelines-toolkit 1.5.1 + // ena-webin-cli 9.0.3 + mgnify-pipelines-toolkit 1.5.5 container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/90/907aee7f61eca630e96a0f50d1bd71d9ba8ac9ca10da02459221930c21076a2a/data': - 'community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:a64d8c87ebf167ef' }" + 'oras://community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:58f85570e2d736cd': + 'community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:2521e265e342b4ec' }" stageInMode 'copy' input: diff --git a/modules/local/registerstudy/environment.yml b/modules/local/registerstudy/environment.yml index bd8ce96b..e03f79e6 100644 --- a/modules/local/registerstudy/environment.yml +++ b/modules/local/registerstudy/environment.yml @@ -5,4 +5,4 @@ channels: - bioconda dependencies: - "conda-forge::python>=3.12" - - "bioconda::mgnify-pipelines-toolkit=1.5.1" + - "bioconda::mgnify-pipelines-toolkit=1.5.5" diff --git a/modules/local/registerstudy/main.nf b/modules/local/registerstudy/main.nf index 2eb83781..0a20f5aa 100644 --- a/modules/local/registerstudy/main.nf +++ b/modules/local/registerstudy/main.nf @@ -4,8 +4,8 @@ process REGISTERSTUDY { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1': - 'biocontainers/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0 ': + 'biocontainers/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0' }" // ENA_WEBIN and ENA_WEBIN_PASSWORD must be set in the process environment. // In the pipeline, map Nextflow secrets via conf/modules.config or nextflow.config: diff --git a/nextflow.config b/nextflow.config index 1d00c41c..dcfeb4c0 100644 --- a/nextflow.config +++ b/nextflow.config @@ -257,6 +257,7 @@ profiles { test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' } test_reads_paired { includeConfig 'conf/test_reads_paired.config' } test_reads_long_bam { includeConfig 'conf/test_reads_long_bam.config' } + test_reads_long_bam_validate { includeConfig 'conf/test_reads_long_bam_validate.config' } } // Load nf-core custom profiles from different institutions diff --git a/tests/assembly_complete_metadata.nf.test.snap b/tests/assembly_complete_metadata.nf.test.snap index 1221e619..45ea7056 100644 --- a/tests/assembly_complete_metadata.nf.test.snap +++ b/tests/assembly_complete_metadata.nf.test.snap @@ -23,12 +23,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_complete_metadata_existing_study.nf.test.snap b/tests/assembly_complete_metadata_existing_study.nf.test.snap index b2fd1c78..9e251d5d 100644 --- a/tests/assembly_complete_metadata_existing_study.nf.test.snap +++ b/tests/assembly_complete_metadata_existing_study.nf.test.snap @@ -24,8 +24,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_no_coverage_paired_reads.nf.test.snap b/tests/assembly_no_coverage_paired_reads.nf.test.snap index b898bf04..21cec5d3 100644 --- a/tests/assembly_no_coverage_paired_reads.nf.test.snap +++ b/tests/assembly_no_coverage_paired_reads.nf.test.snap @@ -26,12 +26,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_no_coverage_single_reads.nf.test.snap b/tests/assembly_no_coverage_single_reads.nf.test.snap index 539dd28f..7dbf494e 100644 --- a/tests/assembly_no_coverage_single_reads.nf.test.snap +++ b/tests/assembly_no_coverage_single_reads.nf.test.snap @@ -26,12 +26,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 10072b0f..fd8cda70 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -41,15 +41,15 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "RENAME_FASTA_FOR_CATPACK": { "bash": "5.2.37(1)-release" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "TRNASCANSE": { "tRNAscan-SE": "2.0.12" diff --git a/tests/mag_complete_metadata.nf.test.snap b/tests/mag_complete_metadata.nf.test.snap index db109eff..497b56cc 100644 --- a/tests/mag_complete_metadata.nf.test.snap +++ b/tests/mag_complete_metadata.nf.test.snap @@ -23,12 +23,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_complete_metadata_existing_study.nf.test.snap b/tests/mag_complete_metadata_existing_study.nf.test.snap index 39b86531..4c21b884 100644 --- a/tests/mag_complete_metadata_existing_study.nf.test.snap +++ b/tests/mag_complete_metadata_existing_study.nf.test.snap @@ -24,8 +24,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_coverage_paired_reads.nf.test.snap b/tests/mag_no_coverage_paired_reads.nf.test.snap index a3b68e51..a8d4989c 100644 --- a/tests/mag_no_coverage_paired_reads.nf.test.snap +++ b/tests/mag_no_coverage_paired_reads.nf.test.snap @@ -26,12 +26,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_coverage_single_reads.nf.test.snap b/tests/mag_no_coverage_single_reads.nf.test.snap index f56b7274..f2fa2e8a 100644 --- a/tests/mag_no_coverage_single_reads.nf.test.snap +++ b/tests/mag_no_coverage_single_reads.nf.test.snap @@ -26,12 +26,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_rna_presence.nf.test.snap b/tests/mag_no_rna_presence.nf.test.snap index e6349e8b..24b5fb33 100644 --- a/tests/mag_no_rna_presence.nf.test.snap +++ b/tests/mag_no_rna_presence.nf.test.snap @@ -1,7 +1,7 @@ { "-profile test_mag_no_rna_presence": { "content": [ - 11, + 17, { "BARRNAP": { "barrnap": 0.9 @@ -29,12 +29,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "TRNASCANSE": { "tRNAscan-SE": "2.0.12" @@ -45,17 +45,28 @@ }, [ "mags", - "mags/no_rna_presence", - "mags/no_rna_presence/rna", - "mags/no_rna_presence/rna/barrnap", - "mags/no_rna_presence/rna/barrnap/no_rna_presence_bac.gff", - "mags/no_rna_presence/rna/trnascanse", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.log", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.stats", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.tsv", - "mags/no_rna_presence/upload", - "mags/no_rna_presence/upload/webin_cli", - "mags/no_rna_presence/upload/webin_cli/no_rna_presence_accessions.tsv", + "mags/no_rna_presence_euk", + "mags/no_rna_presence_euk/rna", + "mags/no_rna_presence_euk/rna/barrnap", + "mags/no_rna_presence_euk/rna/barrnap/no_rna_presence_euk_bac.gff", + "mags/no_rna_presence_euk/rna/trnascanse", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.log", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.stats", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.tsv", + "mags/no_rna_presence_euk/upload", + "mags/no_rna_presence_euk/upload/webin_cli", + "mags/no_rna_presence_euk/upload/webin_cli/no_rna_presence_euk_accessions.tsv", + "mags/no_rna_presence_prok", + "mags/no_rna_presence_prok/rna", + "mags/no_rna_presence_prok/rna/barrnap", + "mags/no_rna_presence_prok/rna/barrnap/no_rna_presence_prok_bac.gff", + "mags/no_rna_presence_prok/rna/trnascanse", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.log", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.stats", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.tsv", + "mags/no_rna_presence_prok/upload", + "mags/no_rna_presence_prok/upload/webin_cli", + "mags/no_rna_presence_prok/upload/webin_cli/no_rna_presence_prok_accessions.tsv", "mags/upload", "mags/upload/genomes_accessions.tsv", "mags/upload/genomes_metadata.tsv", @@ -96,17 +107,19 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "no_rna_presence_bac.gff:md5,df19e1b84ba6f691d20c72b397c88abf", - "no_rna_presence.tsv:md5,abd958e8addba39c9e4d7f6b97a1a7c6", - "genomes_metadata.tsv:md5,28fdaa0fe237cba4d0370c07d5f9ebaa", + "no_rna_presence_euk_bac.gff:md5,df19e1b84ba6f691d20c72b397c88abf", + "no_rna_presence_euk.tsv:md5,68d0b807755e195d09aaa082d59827e7", + "no_rna_presence_prok_bac.gff:md5,468c2be7060d743f95295f1ecf751e19", + "no_rna_presence_prok.tsv:md5,23c19ff33ce36f041803b5b3e7b7681b", + "genomes_metadata.tsv:md5,2acbde2a328213ee6b5153bffa72d697", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_genome_metadata.txt:md5,efa903988140642f4f3d12b8e034e7a1" + "multiqc_genome_metadata.txt:md5,40f68f54d23fdb12371b9b69991101ea" ] ], - "timestamp": "2026-10-19T15:57:54.825156", + "timestamp": "2026-10-26T20:52:59.848025", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.1" + "nextflow": "26.04.0" } } } \ No newline at end of file diff --git a/tests/mag_no_taxonomy.nf.test.snap b/tests/mag_no_taxonomy.nf.test.snap index 725912d2..5a4d987c 100644 --- a/tests/mag_no_taxonomy.nf.test.snap +++ b/tests/mag_no_taxonomy.nf.test.snap @@ -32,15 +32,15 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "RENAME_FASTA_FOR_CATPACK": { "bash": "5.2.37(1)-release" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/reads_long_bam.nf.test b/tests/reads_long_bam.nf.test index 661bf508..81c2c0b4 100644 --- a/tests/reads_long_bam.nf.test +++ b/tests/reads_long_bam.nf.test @@ -32,7 +32,34 @@ nextflow_pipeline { stable_name, // All files with stable contents stable_path - ).match() } + ).match() }, + { + // CREATE_READS_MANIFEST output is not published, so look it up in the work dir + // (skip symlinks staged into downstream tasks) + def manifests = [] + new File("$workDir").eachFileRecurse(groovy.io.FileType.FILES) { f -> + if (f.name.endsWith(".manifest") && !java.nio.file.Files.isSymbolicLink(f.toPath())) { + manifests << f + } + } + assert manifests.size() == 1 + def lines = manifests[0].readLines() + // 8 required fields + DESCRIPTION + 1 BAM entry = 10 lines + assert lines.size() == 10 + // Study is registered on each run, so the accession is not stable + assert lines.any { it ==~ /STUDY\tPRJEB\d+/ } + assert lines.contains("SAMPLE\tSAMEA104188781") + // NAME has a timestamp suffix — check the base ID is present + assert lines.any { it.startsWith("NAME\tbacteroides_fragilis_ont_bam_") } + assert lines.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.contains("INSTRUMENT\tMinION") + assert lines.contains("LIBRARY_SOURCE\tMETAGENOMIC") + assert lines.contains("LIBRARY_SELECTION\tRANDOM") + assert lines.contains("LIBRARY_STRATEGY\tWGS") + assert lines.contains("DESCRIPTION\tTiny unaligned ONT BAM test") + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } ) } } diff --git a/tests/reads_long_bam.nf.test.snap b/tests/reads_long_bam.nf.test.snap new file mode 100644 index 00000000..82d5cd09 --- /dev/null +++ b/tests/reads_long_bam.nf.test.snap @@ -0,0 +1,57 @@ +{ + "-profile test_reads_long_bam": { + "content": [ + 5, + { + "CONCAT_ACCESSIONS": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "CREATE_READS_MANIFEST": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "REGISTERSTUDY": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml", + "reads", + "reads/bacteroides_fragilis_ont_bam", + "reads/bacteroides_fragilis_ont_bam/upload", + "reads/bacteroides_fragilis_ont_bam/upload/webin_cli", + "reads/bacteroides_fragilis_ont_bam/upload/webin_cli/bacteroides_fragilis_ont_bam_accessions.tsv", + "reads/upload", + "reads/upload/reads_accessions.tsv" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-26T19:25:01.401474", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/tests/reads_long_bam_validate.nf.test b/tests/reads_long_bam_validate.nf.test new file mode 100644 index 00000000..a4f5b632 --- /dev/null +++ b/tests/reads_long_bam_validate.nf.test @@ -0,0 +1,65 @@ +nextflow_pipeline { + + name "Test reads submission workflow - long-read BAM (validate)" + script "../main.nf" + tag "pipeline" + tag "mode_reads" + tag "test_reads_long_bam_validate" + profile "test_reads_long_bam_validate" + + test("-profile test_reads_long_bam_validate") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() }, + { + // CREATE_READS_MANIFEST output is not published, so look it up in the work dir + // (skip symlinks staged into downstream tasks) + def manifests = [] + new File("$workDir").eachFileRecurse(groovy.io.FileType.FILES) { f -> + if (f.name.endsWith(".manifest") && !java.nio.file.Files.isSymbolicLink(f.toPath())) { + manifests << f + } + } + assert manifests.size() == 1 + def lines = manifests[0].readLines() + // 8 required fields + DESCRIPTION + 1 BAM entry = 10 lines + assert lines.size() == 10 + assert lines.contains("STUDY\tPRJEB98843") + assert lines.contains("SAMPLE\tSAMEA104188781") + // NAME has a timestamp suffix — check the base ID is present + assert lines.any { it.startsWith("NAME\tbacteroides_fragilis_ont_bam_") } + assert lines.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.contains("INSTRUMENT\tMinION") + assert lines.contains("LIBRARY_SOURCE\tMETAGENOMIC") + assert lines.contains("LIBRARY_SELECTION\tRANDOM") + assert lines.contains("LIBRARY_STRATEGY\tWGS") + assert lines.contains("DESCRIPTION\tTiny unaligned ONT BAM test") + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } + ) + } + } +} diff --git a/tests/reads_long_bam_validate.nf.test.snap b/tests/reads_long_bam_validate.nf.test.snap new file mode 100644 index 00000000..7e9ffdc0 --- /dev/null +++ b/tests/reads_long_bam_validate.nf.test.snap @@ -0,0 +1,42 @@ +{ + "-profile test_reads_long_bam_validate": { + "content": [ + 3, + { + "CREATE_READS_MANIFEST": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-26T19:29:14.217861", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/tests/reads_paired_end.nf.test.snap b/tests/reads_paired_end.nf.test.snap index 7df80d1f..5fe64aca 100644 --- a/tests/reads_paired_end.nf.test.snap +++ b/tests/reads_paired_end.nf.test.snap @@ -9,15 +9,15 @@ "pigz": 2.8 }, "CREATE_READS_MANIFEST": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" From d028562dd2b5e32b193885f22a974b080a85bc68 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Tue, 27 Oct 2026 10:11:04 +0000 Subject: [PATCH 4/6] add -test upload to validate test --- conf/test_reads_long_bam_validate.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test_reads_long_bam_validate.config b/conf/test_reads_long_bam_validate.config index f74f20fa..d2a7a414 100644 --- a/conf/test_reads_long_bam_validate.config +++ b/conf/test_reads_long_bam_validate.config @@ -31,5 +31,5 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" - test_upload = false + test_upload = true } From 1f83a8180d3f6c8e2a97ba24deb31a9b0b020207 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Tue, 27 Oct 2026 14:36:13 +0000 Subject: [PATCH 5/6] apply wording suggestions --- conf/test_reads_long_bam.config | 4 ++-- conf/test_reads_long_bam_validate.config | 4 ++-- docs/usage.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/test_reads_long_bam.config b/conf/test_reads_long_bam.config index d510081d..a0dca38d 100644 --- a/conf/test_reads_long_bam.config +++ b/conf/test_reads_long_bam.config @@ -19,8 +19,8 @@ process { } params { - config_profile_name = 'Test --mode reads long-read BAM profile' - config_profile_description = 'Minimal test profile for long-read BAM reads submission' + config_profile_name = 'Test --mode reads with long reads in BAM format' + config_profile_description = 'Minimal test profile for submission of long reads in BAM format' // Input data input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' diff --git a/conf/test_reads_long_bam_validate.config b/conf/test_reads_long_bam_validate.config index d2a7a414..6d5ff0a1 100644 --- a/conf/test_reads_long_bam_validate.config +++ b/conf/test_reads_long_bam_validate.config @@ -19,8 +19,8 @@ process { } params { - config_profile_name = 'Test --mode reads long-read BAM profile validate only' - config_profile_description = 'Minimal test profile for long-read BAM reads submission' + config_profile_name = 'Test --mode reads with long reads in BAM format (validate only)' + config_profile_description = 'Minimal test profile for submission of long reads in BAM format (validation only)' // Input data input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' diff --git a/docs/usage.md b/docs/usage.md index da9fae4d..a8ab3c53 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -167,7 +167,7 @@ nanopore_run_001,SAMEA7654322,,,data/nanopore_reads.bam,OXFORD_NANOPORE,MinION,M | `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | | `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | | `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required unless `bam` is provided. | -| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads and if `bam` is provided. | +| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads or if `bam` is provided. | | `bam` | Conditional | Path to reads in BAM format (`.bam`). Required unless `fastq_1` is provided. | | `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | | `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | From 8f2b4e61ed2dd223342203e9f79e9f07d9c10701 Mon Sep 17 00:00:00 2001 From: Ekaterina Sakharova Date: Tue, 27 Oct 2026 20:42:44 +0000 Subject: [PATCH 6/6] add assigned accessions to multiqc and update snapshots --- CHANGELOG.md | 1 + assets/multiqc_config.yml | 26 ++++++++++++++++++++++++++ docs/output.md | 2 +- tests/.nftignore | 1 + tests/reads_long_bam.nf.test.snap | 12 ++++++++++-- tests/reads_paired_end.nf.test.snap | 12 ++++++++++-- workflows/readsubmit.nf | 13 ++++++++++++- 7 files changed, 61 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20abedeb..f8566704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#117](https://github.com/nf-core/seqsubmit/pull/117) and [#119](https://github.com/nf-core/seqsubmit/pull/119) - Added BAM files support as raw reads - Added tests for BAM submission with `-submit` and `-validate` modes + - Added assigned `run accession` and `experiment accession` to MultiQC report ([Issue 70](https://github.com/nf-core/seqsubmit/issues/70) resolved) - [#116](https://github.com/nf-core/seqsubmit/pull/116) - All tests except the `*_existing_study` ones now register a new submission study, so they can be run with any Webin account ([#95](https://github.com/nf-core/seqsubmit/issues/95)). The `test_mag_complete_metadata_existing_study` and `test_assembly_complete_metadata_existing_study` tests submit to the existing study `PRJEB98843` and require maintainer Webin credentials - `test_upload = true` is set explicitly in all test configs diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index c8b1a898..f0f18f71 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -21,6 +21,7 @@ custom_content: - sample_registration - submission_results_assemblies - submission_results_genomes + - submission_results_reads sp: assembly_metadata: @@ -33,6 +34,8 @@ sp: fn_re: 'genomes_accessions\.tsv$' submission_results_assemblies: fn_re: 'assemblies_accessions\.tsv$' + submission_results_reads: + fn_re: 'reads_accessions_split\.tsv$' # Custom data configuration custom_data: @@ -160,3 +163,26 @@ custom_data: accession: title: "ENA accession" description: "Assigned accession after submission" + + submission_results_reads: + id: "submission_results_reads" + section_name: "Submission results" + description: > + Accessions assigned during upload process (experiment and run accessions). + Users should read the ENA documentation on referencing submitted data: + https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html + file_format: "tsv" + plot_type: "table" + pconfig: + id: "Reads assigned accessions" + title: "Submission results" + headers: + alias: + title: "alias" + description: "Read submission name" + experiment: + title: "ENA experiment" + description: "Experiment accession (ERX) assigned after submission" + run: + title: "ENA run" + description: "Run accession (ERR) assigned after submission" diff --git a/docs/output.md b/docs/output.md index 50a28c12..438cf91a 100644 --- a/docs/output.md +++ b/docs/output.md @@ -67,7 +67,7 @@ When `--mode reads` is used, results are written under `reads/`. Output files - `reads/` - - `upload/reads_accessions.tsv`: run accessions assigned to submitted reads. + - `upload/reads_accessions.tsv`: experiment and run accessions assigned to submitted reads (also shown in the MultiQC report). diff --git a/tests/.nftignore b/tests/.nftignore index 4e6fdc18..f2bc9b9c 100644 --- a/tests/.nftignore +++ b/tests/.nftignore @@ -7,6 +7,7 @@ multiqc/multiqc_data/multiqc_software_versions.txt multiqc/multiqc_data/llms-full.txt multiqc/multiqc_data/multiqc_Assemblies assigned accessions.txt multiqc/multiqc_data/multiqc_Genomes assigned accessions.txt +multiqc/multiqc_data/multiqc_Reads assigned accessions.txt multiqc/multiqc_data/multiqc_sample_registration.txt multiqc/multiqc_plots/{svg,pdf,png}/*.{svg,pdf,png} multiqc/multiqc_report.html diff --git a/tests/reads_long_bam.nf.test.snap b/tests/reads_long_bam.nf.test.snap index 82d5cd09..20f9bec3 100644 --- a/tests/reads_long_bam.nf.test.snap +++ b/tests/reads_long_bam.nf.test.snap @@ -29,10 +29,18 @@ "multiqc/multiqc_data/llms-full.txt", "multiqc/multiqc_data/multiqc.log", "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Reads assigned accessions.txt", "multiqc/multiqc_data/multiqc_citations.txt", "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Reads assigned accessions.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Reads assigned accessions.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Reads assigned accessions.svg", "multiqc/multiqc_report.html", "pipeline_info", "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml", @@ -48,10 +56,10 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-26T19:25:01.401474", + "timestamp": "2026-10-27T20:25:58.703414", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.0" + "nextflow": "26.10.4" } } } \ No newline at end of file diff --git a/tests/reads_paired_end.nf.test.snap b/tests/reads_paired_end.nf.test.snap index 5fe64aca..efb617ae 100644 --- a/tests/reads_paired_end.nf.test.snap +++ b/tests/reads_paired_end.nf.test.snap @@ -29,10 +29,18 @@ "multiqc/multiqc_data/llms-full.txt", "multiqc/multiqc_data/multiqc.log", "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Reads assigned accessions.txt", "multiqc/multiqc_data/multiqc_citations.txt", "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Reads assigned accessions.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Reads assigned accessions.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Reads assigned accessions.svg", "multiqc/multiqc_report.html", "pipeline_info", "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml", @@ -48,10 +56,10 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-19T15:18:28.953324", + "timestamp": "2026-10-27T20:27:33.457306", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.1" + "nextflow": "26.10.4" } } } \ No newline at end of file diff --git a/workflows/readsubmit.nf b/workflows/readsubmit.nf index e2a3fa30..6e94adb8 100644 --- a/workflows/readsubmit.nf +++ b/workflows/readsubmit.nf @@ -141,7 +141,18 @@ workflow READSUBMIT { : file("${projectDir}/assets/methods_description_template.yml", checkIfExists: true) def ch_methods_description = channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: true)) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{_meta, file -> file}) + // Split "ERX...,ERR..." accession column into experiment and run columns for the report + def ch_reads_accessions_split = CONCAT_ACCESSIONS.out.file_out + .flatMap { _meta, tsv -> tsv.readLines().drop(1).findAll { line -> line.trim() } } + .map { line -> + def (alias, accessions) = line.tokenize('\t') + [''] + def accession_list = accessions.tokenize(',')*.trim() + def experiment = accession_list.find { acc -> acc ==~ /^[EDS]RX\d+$/ } ?: '' + def run = accession_list.find { acc -> acc ==~ /^[EDS]RR\d+$/ } ?: '' + "${alias}\t${experiment}\t${run}" + } + .collectFile(name: 'reads_accessions_split.tsv', seed: "alias\texperiment\trun", newLine: true, sort: true) + ch_multiqc_files = ch_multiqc_files.mix(ch_reads_accessions_split) MULTIQC( ch_multiqc_files.flatten().collect().map { files -> [