diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b9d142..f8566704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#117](https://github.com/nf-core/seqsubmit/pull/117) and [#119](https://github.com/nf-core/seqsubmit/pull/119) + - Added BAM files support as raw reads + - Added tests for BAM submission with `-submit` and `-validate` modes + - Added assigned `run accession` and `experiment accession` to MultiQC report ([Issue 70](https://github.com/nf-core/seqsubmit/issues/70) resolved) - [#116](https://github.com/nf-core/seqsubmit/pull/116) - All tests except the `*_existing_study` ones now register a new submission study, so they can be run with any Webin account ([#95](https://github.com/nf-core/seqsubmit/issues/95)). The `test_mag_complete_metadata_existing_study` and `test_assembly_complete_metadata_existing_study` tests submit to the existing study `PRJEB98843` and require maintainer Webin credentials - `test_upload = true` is set explicitly in all test configs diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index c8b1a898..f0f18f71 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -21,6 +21,7 @@ custom_content: - sample_registration - submission_results_assemblies - submission_results_genomes + - submission_results_reads sp: assembly_metadata: @@ -33,6 +34,8 @@ sp: fn_re: 'genomes_accessions\.tsv$' submission_results_assemblies: fn_re: 'assemblies_accessions\.tsv$' + submission_results_reads: + fn_re: 'reads_accessions_split\.tsv$' # Custom data configuration custom_data: @@ -160,3 +163,26 @@ custom_data: accession: title: "ENA accession" description: "Assigned accession after submission" + + submission_results_reads: + id: "submission_results_reads" + section_name: "Submission results" + description: > + Accessions assigned during upload process (experiment and run accessions). + Users should read the ENA documentation on referencing submitted data: + https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html + file_format: "tsv" + plot_type: "table" + pconfig: + id: "Reads assigned accessions" + title: "Submission results" + headers: + alias: + title: "alias" + description: "Read submission name" + experiment: + title: "ENA experiment" + description: "Experiment accession (ERX) assigned after submission" + run: + title: "ENA run" + description: "Run accession (ERR) assigned after submission" diff --git a/assets/schema_input_reads.json b/assets/schema_input_reads.json index c511c3cc..963fe107 100644 --- a/assets/schema_input_reads.json +++ b/assets/schema_input_reads.json @@ -22,12 +22,21 @@ "meta": ["sample_accession"] }, "fastq_1": { - "type": "string", - "format": "file-path", - "exists": true, - "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$", + "anyOf": [ + { + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped)" + "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Leave empty if reads are provided as BAM" }, "fastq_2": { "anyOf": [ @@ -46,6 +55,23 @@ "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads" }, + "bam": { + "anyOf": [ + { + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.bam$" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "errorMessage": "BAM file must have extension '.bam'", + "description": "Path to reads in unaligned or aligned BAM format. Use instead of fastq_1/fastq_2" + }, "platform": { "type": "string", "enum": [ @@ -223,12 +249,53 @@ "required": [ "id", "sample_accession", - "fastq_1", "platform", "instrument", "library_source", "library_selection", "library_strategy" - ] + ], + "oneOf": [ + { + "properties": { + "fastq_1": { + "type": "string", + "minLength": 1 + } + }, + "required": ["fastq_1"] + }, + { + "properties": { + "bam": { + "type": "string", + "minLength": 1 + } + }, + "required": ["bam"] + } + ], + "if": { + "properties": { + "bam": { + "type": "string", + "minLength": 1 + } + }, + "required": ["bam"] + }, + "then": { + "properties": { + "fastq_1": { + "maxLength": 0, + "errorMessage": "fastq_1 must be empty when reads are provided as BAM" + }, + "fastq_2": { + "maxLength": 0, + "errorMessage": "fastq_2 must be empty when reads are provided as BAM" + } + } + }, + "errorMessage": "Reads must be provided either as FASTQ (fastq_1, optionally fastq_2) or as a single BAM (bam, with fastq_1 and fastq_2 empty)" } } diff --git a/conf/modules.config b/conf/modules.config index 915a09c2..c14d7f2c 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -176,6 +176,12 @@ process { ] } + // Reads can be multi or 100s of GB (e.g. long-read BAMs): symlink instead of copying them before upload. + // assemblies, bins and mags modes keep the default stageInMode 'copy'. + withName: '.*:READSUBMIT:SUBMIT' { + stageInMode = 'symlink' + } + withName: 'REGISTERSTUDY|CREATE_READS_MANIFEST' { publishDir = [ enabled: false diff --git a/conf/test_reads_long_bam.config b/conf/test_reads_long_bam.config new file mode 100644 index 00000000..a0dca38d --- /dev/null +++ b/conf/test_reads_long_bam.config @@ -0,0 +1,36 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_reads_long_bam, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode reads with long reads in BAM format' + config_profile_description = 'Minimal test profile for submission of long reads in BAM format' + + // Input data + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' + outdir = 'test_output' + + mode = "reads" + webincli_mode = "submit" + submission_study = null + study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json' + centre_name = "TEST_CENTER" + + test_upload = true +} diff --git a/conf/test_reads_long_bam_validate.config b/conf/test_reads_long_bam_validate.config new file mode 100644 index 00000000..6d5ff0a1 --- /dev/null +++ b/conf/test_reads_long_bam_validate.config @@ -0,0 +1,35 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_reads_long_bam_validate, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode reads with long reads in BAM format (validate only)' + config_profile_description = 'Minimal test profile for submission of long reads in BAM format (validation only)' + + // Input data + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv' + outdir = 'test_output' + + mode = "reads" + webincli_mode = "validate" + submission_study = "PRJEB98843" + centre_name = "TEST_CENTER" + + test_upload = true +} diff --git a/docs/output.md b/docs/output.md index 50a28c12..438cf91a 100644 --- a/docs/output.md +++ b/docs/output.md @@ -67,7 +67,7 @@ When `--mode reads` is used, results are written under `reads/`. Output files - `reads/` - - `upload/reads_accessions.tsv`: run accessions assigned to submitted reads. + - `upload/reads_accessions.tsv`: experiment and run accessions assigned to submitted reads (also shown in the MultiQC report). diff --git a/docs/usage.md b/docs/usage.md index efd25471..a8ab3c53 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -156,25 +156,27 @@ You will need to create a samplesheet with information about the raw reads you w Example: ```csv title="samplesheet_reads.csv" -id,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description -illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ -pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing +id,sample_accession,fastq_1,fastq_2,bam,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description +illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ +pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing +nanopore_run_001,SAMEA7654322,,,data/nanopore_reads.bam,OXFORD_NANOPORE,MinION,METAGENOMIC,RANDOM,WGS,,ONT_library_003,Long-read sequencing provided as BAM ``` -| Column | Required | Description | -| ------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | -| `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | -| `fastq_1` | Yes | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). | -| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads. | -| `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | -| `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | -| `library_source` | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. ENA [documantation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source). | -| `library_selection` | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection) for complete list. | -| `library_strategy` | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy) for complete list. | -| `insert_size` | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | -| `library_name` | No | Descriptive library name (optional). | -| `description` | No | Free-text description of the experiment (optional). | +| Column | Required | Description | +| ------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | +| `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | +| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required unless `bam` is provided. | +| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads or if `bam` is provided. | +| `bam` | Conditional | Path to reads in BAM format (`.bam`). Required unless `fastq_1` is provided. | +| `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | +| `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | +| `library_source` | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. ENA [documantation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source). | +| `library_selection` | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection) for complete list. | +| `library_strategy` | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy) for complete list. | +| `insert_size` | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | +| `library_name` | No | Descriptive library name (optional). | +| `description` | No | Free-text description of the experiment (optional). | ### `reads` parameters and example @@ -197,7 +199,8 @@ nextflow run nf-core/seqsubmit \ `reads` mode does not support the following: -- **Reads in non-FASTQ formats**: Only reads provided as FASTQ (`.fastq.gz`) can be submitted. Other formats (e.g. BAM) are not supported as input. +- **Other read formats**: Reads must be provided as FASTQ (`.fastq`/`.fq`, optionally gzipped) or BAM (`.bam`). Other formats (e.g. CRAM or `.tar.gz` archives) are not supported as input. +- **Mixed formats in one row**: A row cannot combine FASTQ and BAM files. Put them in separate rows, which are submitted as separate ENA runs. ## `metagenomic_assemblies` mode diff --git a/modules/local/create_reads_manifest/environment.yml b/modules/local/create_reads_manifest/environment.yml index 053a4fb6..21d90955 100644 --- a/modules/local/create_reads_manifest/environment.yml +++ b/modules/local/create_reads_manifest/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda dependencies: - conda-forge::python>=3.12 - - bioconda::mgnify-pipelines-toolkit=1.5.1 + - bioconda::mgnify-pipelines-toolkit=1.5.5 diff --git a/modules/local/create_reads_manifest/main.nf b/modules/local/create_reads_manifest/main.nf index ded09e79..dba90d3a 100644 --- a/modules/local/create_reads_manifest/main.nf +++ b/modules/local/create_reads_manifest/main.nf @@ -4,11 +4,11 @@ process CREATE_READS_MANIFEST { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1': - 'biocontainers/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0 ': + 'biocontainers/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0' }" input: - tuple val(meta), path(fastq_files) + tuple val(meta), path(read_files) val(study_accession) output: @@ -21,8 +21,9 @@ process CREATE_READS_MANIFEST { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def fastq_list = fastq_files instanceof List ? fastq_files : [fastq_files] - def fastq_args = fastq_list.collect { "--fastq ${it.name}" }.join(' \\\n ') + def read_list = read_files instanceof List ? read_files : [read_files] + def read_flag = meta.file_type == 'bam' ? '--bam' : '--fastq' + def read_args = read_list.collect { file -> "${read_flag} ${file.name}" }.join(' \\\n ') def opt_insert = meta.insert_size ? "--insert-size ${meta.insert_size}" : '' def opt_lib_name = meta.library_name ? "--library-name '${meta.library_name}'" : '' def opt_desc = meta.description ? "--description '${meta.description}'" : '' @@ -38,7 +39,7 @@ process CREATE_READS_MANIFEST { --library-selection '${meta.library_selection}' \\ --library-strategy '${meta.library_strategy}' \\ --output ${prefix}.manifest \\ - ${fastq_args} \\ + ${read_args} \\ ${opt_insert} \\ ${opt_lib_name} \\ ${opt_desc} \\ diff --git a/modules/local/create_reads_manifest/meta.yml b/modules/local/create_reads_manifest/meta.yml index b8bead49..b95ff690 100644 --- a/modules/local/create_reads_manifest/meta.yml +++ b/modules/local/create_reads_manifest/meta.yml @@ -29,13 +29,15 @@ input: description: | Groovy Map containing sample information, including platform, instrument, library_source, library_selection, library_strategy, - sample_accession, and optionally insert_size, library_name, - description. - e.g. `[ id:'sample1', platform:'ILLUMINA', instrument:'Illumina HiSeq 2500' ]` - - fastq_files: + sample_accession, file_type ('fastq' or 'bam'), and optionally + insert_size, library_name, description. + e.g. `[ id:'sample1', platform:'ILLUMINA', instrument:'Illumina HiSeq 2500', file_type:'fastq' ]` + - read_files: type: file - description: One or two FASTQ files (single-end or paired-end reads). - pattern: "*.{fastq,fastq.gz,fq,fq.gz}" + description: | + One or two FASTQ files (single-end or paired-end reads), or a single + BAM file. Written to the manifest as FASTQ or BAM according to meta.file_type. + pattern: "*.{fastq,fastq.gz,fq,fq.gz,bam}" - - study_accession: type: value description: ENA study accession to submit reads to. diff --git a/modules/local/create_reads_manifest/tests/main.nf.test b/modules/local/create_reads_manifest/tests/main.nf.test index 3c33494c..56439fe7 100644 --- a/modules/local/create_reads_manifest/tests/main.nf.test +++ b/modules/local/create_reads_manifest/tests/main.nf.test @@ -96,4 +96,47 @@ nextflow_process { } } + test("CREATE_READS_MANIFEST - long-read BAM") { + when { + process { + """ + input[0] = [ + [ + id: 'test_bam', + sample_accession: 'SAMEA12345', + platform: 'OXFORD_NANOPORE', + instrument: 'MinION', + library_source: 'METAGENOMIC', + library_selection: 'RANDOM', + library_strategy: 'WGS', + insert_size: null, + library_name: null, + description: null, + file_type: 'bam' + ], + file(params.pipelines_testdata_base_path + 'test_data/reads/tiny_ont.bam', checkIfExists: true) + ] + input[1] = "PRJEB12345" + """ + } + } + + then { + assert process.success + assertAll( + { assert process.out.manifest.size() == 1 }, + { + def manifestContent = path(process.out.manifest[0][1]).text + def lines = manifestContent.readLines() + // 8 required fields + 1 BAM entry = 9 lines + assert lines.size() == 9 + assert manifestContent.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.count { it.startsWith("BAM\t") } == 1 + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } + ) + } + } + } diff --git a/modules/local/ena_webin_cli_wrapper/environment.yml b/modules/local/ena_webin_cli_wrapper/environment.yml index fe7d46d3..93d49b8d 100644 --- a/modules/local/ena_webin_cli_wrapper/environment.yml +++ b/modules/local/ena_webin_cli_wrapper/environment.yml @@ -3,6 +3,6 @@ channels: - conda-forge - bioconda dependencies: - - "bioconda::mgnify-pipelines-toolkit=1.5.1" + - "bioconda::mgnify-pipelines-toolkit=1.5.5" - "conda-forge::openjdk" - "bioconda::ena-webin-cli=9.0.3" diff --git a/modules/local/ena_webin_cli_wrapper/main.nf b/modules/local/ena_webin_cli_wrapper/main.nf index 7a689619..2ac07077 100644 --- a/modules/local/ena_webin_cli_wrapper/main.nf +++ b/modules/local/ena_webin_cli_wrapper/main.nf @@ -8,10 +8,10 @@ process ENA_WEBIN_CLI_WRAPPER { conda "${moduleDir}/environment.yml" - // ena-webin-cli 9.0.3 + mgnify-pipelines-toolkit 1.5.1 + // ena-webin-cli 9.0.3 + mgnify-pipelines-toolkit 1.5.5 container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/90/907aee7f61eca630e96a0f50d1bd71d9ba8ac9ca10da02459221930c21076a2a/data': - 'community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:a64d8c87ebf167ef' }" + 'oras://community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:58f85570e2d736cd': + 'community.wave.seqera.io/library/ena-webin-cli_mgnify-pipelines-toolkit:2521e265e342b4ec' }" stageInMode 'copy' input: diff --git a/modules/local/registerstudy/environment.yml b/modules/local/registerstudy/environment.yml index bd8ce96b..e03f79e6 100644 --- a/modules/local/registerstudy/environment.yml +++ b/modules/local/registerstudy/environment.yml @@ -5,4 +5,4 @@ channels: - bioconda dependencies: - "conda-forge::python>=3.12" - - "bioconda::mgnify-pipelines-toolkit=1.5.1" + - "bioconda::mgnify-pipelines-toolkit=1.5.5" diff --git a/modules/local/registerstudy/main.nf b/modules/local/registerstudy/main.nf index 2eb83781..0a20f5aa 100644 --- a/modules/local/registerstudy/main.nf +++ b/modules/local/registerstudy/main.nf @@ -4,8 +4,8 @@ process REGISTERSTUDY { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1': - 'biocontainers/mgnify-pipelines-toolkit:1.5.1--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0 ': + 'biocontainers/mgnify-pipelines-toolkit:1.5.5--pyhdfd78af_0' }" // ENA_WEBIN and ENA_WEBIN_PASSWORD must be set in the process environment. // In the pipeline, map Nextflow secrets via conf/modules.config or nextflow.config: diff --git a/nextflow.config b/nextflow.config index 9bddb0bb..dcfeb4c0 100644 --- a/nextflow.config +++ b/nextflow.config @@ -239,6 +239,7 @@ profiles { apptainer.runOptions = '--nv' singularity.runOptions = '--nv' } + test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_mag_complete_metadata_existing_study { includeConfig 'conf/test_mag_complete_metadata_existing_study.conf' } @@ -255,6 +256,8 @@ profiles { test_assembly_no_coverage_paired_reads { includeConfig 'conf/test_assembly_no_coverage_paired_reads.config' } test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' } test_reads_paired { includeConfig 'conf/test_reads_paired.config' } + test_reads_long_bam { includeConfig 'conf/test_reads_long_bam.config' } + test_reads_long_bam_validate { includeConfig 'conf/test_reads_long_bam_validate.config' } } // Load nf-core custom profiles from different institutions diff --git a/tests/.nftignore b/tests/.nftignore index 4e6fdc18..f2bc9b9c 100644 --- a/tests/.nftignore +++ b/tests/.nftignore @@ -7,6 +7,7 @@ multiqc/multiqc_data/multiqc_software_versions.txt multiqc/multiqc_data/llms-full.txt multiqc/multiqc_data/multiqc_Assemblies assigned accessions.txt multiqc/multiqc_data/multiqc_Genomes assigned accessions.txt +multiqc/multiqc_data/multiqc_Reads assigned accessions.txt multiqc/multiqc_data/multiqc_sample_registration.txt multiqc/multiqc_plots/{svg,pdf,png}/*.{svg,pdf,png} multiqc/multiqc_report.html diff --git a/tests/assembly_complete_metadata.nf.test.snap b/tests/assembly_complete_metadata.nf.test.snap index 1221e619..45ea7056 100644 --- a/tests/assembly_complete_metadata.nf.test.snap +++ b/tests/assembly_complete_metadata.nf.test.snap @@ -23,12 +23,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_complete_metadata_existing_study.nf.test.snap b/tests/assembly_complete_metadata_existing_study.nf.test.snap index b2fd1c78..9e251d5d 100644 --- a/tests/assembly_complete_metadata_existing_study.nf.test.snap +++ b/tests/assembly_complete_metadata_existing_study.nf.test.snap @@ -24,8 +24,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_no_coverage_paired_reads.nf.test.snap b/tests/assembly_no_coverage_paired_reads.nf.test.snap index b898bf04..21cec5d3 100644 --- a/tests/assembly_no_coverage_paired_reads.nf.test.snap +++ b/tests/assembly_no_coverage_paired_reads.nf.test.snap @@ -26,12 +26,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_no_coverage_single_reads.nf.test.snap b/tests/assembly_no_coverage_single_reads.nf.test.snap index 539dd28f..7dbf494e 100644 --- a/tests/assembly_no_coverage_single_reads.nf.test.snap +++ b/tests/assembly_no_coverage_single_reads.nf.test.snap @@ -26,12 +26,12 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 10072b0f..fd8cda70 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -41,15 +41,15 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "RENAME_FASTA_FOR_CATPACK": { "bash": "5.2.37(1)-release" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "TRNASCANSE": { "tRNAscan-SE": "2.0.12" diff --git a/tests/mag_complete_metadata.nf.test.snap b/tests/mag_complete_metadata.nf.test.snap index db109eff..497b56cc 100644 --- a/tests/mag_complete_metadata.nf.test.snap +++ b/tests/mag_complete_metadata.nf.test.snap @@ -23,12 +23,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_complete_metadata_existing_study.nf.test.snap b/tests/mag_complete_metadata_existing_study.nf.test.snap index 39b86531..4c21b884 100644 --- a/tests/mag_complete_metadata_existing_study.nf.test.snap +++ b/tests/mag_complete_metadata_existing_study.nf.test.snap @@ -24,8 +24,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_coverage_paired_reads.nf.test.snap b/tests/mag_no_coverage_paired_reads.nf.test.snap index a3b68e51..a8d4989c 100644 --- a/tests/mag_no_coverage_paired_reads.nf.test.snap +++ b/tests/mag_no_coverage_paired_reads.nf.test.snap @@ -26,12 +26,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_coverage_single_reads.nf.test.snap b/tests/mag_no_coverage_single_reads.nf.test.snap index f56b7274..f2fa2e8a 100644 --- a/tests/mag_no_coverage_single_reads.nf.test.snap +++ b/tests/mag_no_coverage_single_reads.nf.test.snap @@ -26,12 +26,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/mag_no_rna_presence.nf.test.snap b/tests/mag_no_rna_presence.nf.test.snap index e6349e8b..24b5fb33 100644 --- a/tests/mag_no_rna_presence.nf.test.snap +++ b/tests/mag_no_rna_presence.nf.test.snap @@ -1,7 +1,7 @@ { "-profile test_mag_no_rna_presence": { "content": [ - 11, + 17, { "BARRNAP": { "barrnap": 0.9 @@ -29,12 +29,12 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "TRNASCANSE": { "tRNAscan-SE": "2.0.12" @@ -45,17 +45,28 @@ }, [ "mags", - "mags/no_rna_presence", - "mags/no_rna_presence/rna", - "mags/no_rna_presence/rna/barrnap", - "mags/no_rna_presence/rna/barrnap/no_rna_presence_bac.gff", - "mags/no_rna_presence/rna/trnascanse", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.log", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.stats", - "mags/no_rna_presence/rna/trnascanse/no_rna_presence.tsv", - "mags/no_rna_presence/upload", - "mags/no_rna_presence/upload/webin_cli", - "mags/no_rna_presence/upload/webin_cli/no_rna_presence_accessions.tsv", + "mags/no_rna_presence_euk", + "mags/no_rna_presence_euk/rna", + "mags/no_rna_presence_euk/rna/barrnap", + "mags/no_rna_presence_euk/rna/barrnap/no_rna_presence_euk_bac.gff", + "mags/no_rna_presence_euk/rna/trnascanse", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.log", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.stats", + "mags/no_rna_presence_euk/rna/trnascanse/no_rna_presence_euk.tsv", + "mags/no_rna_presence_euk/upload", + "mags/no_rna_presence_euk/upload/webin_cli", + "mags/no_rna_presence_euk/upload/webin_cli/no_rna_presence_euk_accessions.tsv", + "mags/no_rna_presence_prok", + "mags/no_rna_presence_prok/rna", + "mags/no_rna_presence_prok/rna/barrnap", + "mags/no_rna_presence_prok/rna/barrnap/no_rna_presence_prok_bac.gff", + "mags/no_rna_presence_prok/rna/trnascanse", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.log", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.stats", + "mags/no_rna_presence_prok/rna/trnascanse/no_rna_presence_prok.tsv", + "mags/no_rna_presence_prok/upload", + "mags/no_rna_presence_prok/upload/webin_cli", + "mags/no_rna_presence_prok/upload/webin_cli/no_rna_presence_prok_accessions.tsv", "mags/upload", "mags/upload/genomes_accessions.tsv", "mags/upload/genomes_metadata.tsv", @@ -96,17 +107,19 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "no_rna_presence_bac.gff:md5,df19e1b84ba6f691d20c72b397c88abf", - "no_rna_presence.tsv:md5,abd958e8addba39c9e4d7f6b97a1a7c6", - "genomes_metadata.tsv:md5,28fdaa0fe237cba4d0370c07d5f9ebaa", + "no_rna_presence_euk_bac.gff:md5,df19e1b84ba6f691d20c72b397c88abf", + "no_rna_presence_euk.tsv:md5,68d0b807755e195d09aaa082d59827e7", + "no_rna_presence_prok_bac.gff:md5,468c2be7060d743f95295f1ecf751e19", + "no_rna_presence_prok.tsv:md5,23c19ff33ce36f041803b5b3e7b7681b", + "genomes_metadata.tsv:md5,2acbde2a328213ee6b5153bffa72d697", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", - "multiqc_genome_metadata.txt:md5,efa903988140642f4f3d12b8e034e7a1" + "multiqc_genome_metadata.txt:md5,40f68f54d23fdb12371b9b69991101ea" ] ], - "timestamp": "2026-10-19T15:57:54.825156", + "timestamp": "2026-10-26T20:52:59.848025", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.1" + "nextflow": "26.04.0" } } } \ No newline at end of file diff --git a/tests/mag_no_taxonomy.nf.test.snap b/tests/mag_no_taxonomy.nf.test.snap index 725912d2..5a4d987c 100644 --- a/tests/mag_no_taxonomy.nf.test.snap +++ b/tests/mag_no_taxonomy.nf.test.snap @@ -32,15 +32,15 @@ "falint": "1.2.0" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "RENAME_FASTA_FOR_CATPACK": { "bash": "5.2.37(1)-release" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/reads_long_bam.nf.test b/tests/reads_long_bam.nf.test new file mode 100644 index 00000000..81c2c0b4 --- /dev/null +++ b/tests/reads_long_bam.nf.test @@ -0,0 +1,66 @@ +nextflow_pipeline { + + name "Test reads submission workflow - long-read BAM" + script "../main.nf" + tag "pipeline" + tag "mode_reads" + tag "test_reads_long_bam" + profile "test_reads_long_bam" + + test("-profile test_reads_long_bam") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() }, + { + // CREATE_READS_MANIFEST output is not published, so look it up in the work dir + // (skip symlinks staged into downstream tasks) + def manifests = [] + new File("$workDir").eachFileRecurse(groovy.io.FileType.FILES) { f -> + if (f.name.endsWith(".manifest") && !java.nio.file.Files.isSymbolicLink(f.toPath())) { + manifests << f + } + } + assert manifests.size() == 1 + def lines = manifests[0].readLines() + // 8 required fields + DESCRIPTION + 1 BAM entry = 10 lines + assert lines.size() == 10 + // Study is registered on each run, so the accession is not stable + assert lines.any { it ==~ /STUDY\tPRJEB\d+/ } + assert lines.contains("SAMPLE\tSAMEA104188781") + // NAME has a timestamp suffix — check the base ID is present + assert lines.any { it.startsWith("NAME\tbacteroides_fragilis_ont_bam_") } + assert lines.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.contains("INSTRUMENT\tMinION") + assert lines.contains("LIBRARY_SOURCE\tMETAGENOMIC") + assert lines.contains("LIBRARY_SELECTION\tRANDOM") + assert lines.contains("LIBRARY_STRATEGY\tWGS") + assert lines.contains("DESCRIPTION\tTiny unaligned ONT BAM test") + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } + ) + } + } +} diff --git a/tests/reads_long_bam.nf.test.snap b/tests/reads_long_bam.nf.test.snap new file mode 100644 index 00000000..20f9bec3 --- /dev/null +++ b/tests/reads_long_bam.nf.test.snap @@ -0,0 +1,65 @@ +{ + "-profile test_reads_long_bam": { + "content": [ + 5, + { + "CONCAT_ACCESSIONS": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "CREATE_READS_MANIFEST": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "REGISTERSTUDY": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Reads assigned accessions.txt", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Reads assigned accessions.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Reads assigned accessions.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Reads assigned accessions.svg", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml", + "reads", + "reads/bacteroides_fragilis_ont_bam", + "reads/bacteroides_fragilis_ont_bam/upload", + "reads/bacteroides_fragilis_ont_bam/upload/webin_cli", + "reads/bacteroides_fragilis_ont_bam/upload/webin_cli/bacteroides_fragilis_ont_bam_accessions.tsv", + "reads/upload", + "reads/upload/reads_accessions.tsv" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-27T20:25:58.703414", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.10.4" + } + } +} \ No newline at end of file diff --git a/tests/reads_long_bam_validate.nf.test b/tests/reads_long_bam_validate.nf.test new file mode 100644 index 00000000..a4f5b632 --- /dev/null +++ b/tests/reads_long_bam_validate.nf.test @@ -0,0 +1,65 @@ +nextflow_pipeline { + + name "Test reads submission workflow - long-read BAM (validate)" + script "../main.nf" + tag "pipeline" + tag "mode_reads" + tag "test_reads_long_bam_validate" + profile "test_reads_long_bam_validate" + + test("-profile test_reads_long_bam_validate") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() }, + { + // CREATE_READS_MANIFEST output is not published, so look it up in the work dir + // (skip symlinks staged into downstream tasks) + def manifests = [] + new File("$workDir").eachFileRecurse(groovy.io.FileType.FILES) { f -> + if (f.name.endsWith(".manifest") && !java.nio.file.Files.isSymbolicLink(f.toPath())) { + manifests << f + } + } + assert manifests.size() == 1 + def lines = manifests[0].readLines() + // 8 required fields + DESCRIPTION + 1 BAM entry = 10 lines + assert lines.size() == 10 + assert lines.contains("STUDY\tPRJEB98843") + assert lines.contains("SAMPLE\tSAMEA104188781") + // NAME has a timestamp suffix — check the base ID is present + assert lines.any { it.startsWith("NAME\tbacteroides_fragilis_ont_bam_") } + assert lines.contains("PLATFORM\tOXFORD_NANOPORE") + assert lines.contains("INSTRUMENT\tMinION") + assert lines.contains("LIBRARY_SOURCE\tMETAGENOMIC") + assert lines.contains("LIBRARY_SELECTION\tRANDOM") + assert lines.contains("LIBRARY_STRATEGY\tWGS") + assert lines.contains("DESCRIPTION\tTiny unaligned ONT BAM test") + assert lines.contains("BAM\ttiny_ont.bam") + assert lines.count { it.startsWith("FASTQ\t") } == 0 + } + ) + } + } +} diff --git a/tests/reads_long_bam_validate.nf.test.snap b/tests/reads_long_bam_validate.nf.test.snap new file mode 100644 index 00000000..7e9ffdc0 --- /dev/null +++ b/tests/reads_long_bam_validate.nf.test.snap @@ -0,0 +1,42 @@ +{ + "-profile test_reads_long_bam_validate": { + "content": [ + 3, + { + "CREATE_READS_MANIFEST": { + "mgnify-pipelines-toolkit": "1.5.5" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-26T19:29:14.217861", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + } +} \ No newline at end of file diff --git a/tests/reads_paired_end.nf.test.snap b/tests/reads_paired_end.nf.test.snap index 7df80d1f..efb617ae 100644 --- a/tests/reads_paired_end.nf.test.snap +++ b/tests/reads_paired_end.nf.test.snap @@ -9,15 +9,15 @@ "pigz": 2.8 }, "CREATE_READS_MANIFEST": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" @@ -29,10 +29,18 @@ "multiqc/multiqc_data/llms-full.txt", "multiqc/multiqc_data/multiqc.log", "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Reads assigned accessions.txt", "multiqc/multiqc_data/multiqc_citations.txt", "multiqc/multiqc_data/multiqc_data.json", "multiqc/multiqc_data/multiqc_software_versions.txt", "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Reads assigned accessions.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Reads assigned accessions.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Reads assigned accessions.svg", "multiqc/multiqc_report.html", "pipeline_info", "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml", @@ -48,10 +56,10 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-19T15:18:28.953324", + "timestamp": "2026-10-27T20:27:33.457306", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.1" + "nextflow": "26.10.4" } } } \ No newline at end of file diff --git a/workflows/readsubmit.nf b/workflows/readsubmit.nf index 8bdff882..6e94adb8 100644 --- a/workflows/readsubmit.nf +++ b/workflows/readsubmit.nf @@ -45,14 +45,16 @@ workflow READSUBMIT { // --------- Create reads channel with proper metadata structure reads_ch = ch_samplesheet - .map { meta, reads_1, reads_2 -> - def new_meta = meta + [single_end: !reads_2] - if ( !new_meta.single_end ) { + .map { meta, reads_1, reads_2, bam -> + if ( bam ) { + // Reads provided as BAM + [meta + [single_end: true, file_type: 'bam'], bam] + } else if ( reads_2 ) { // If paired end reads - [new_meta, [reads_1, reads_2]] + [meta + [single_end: false, file_type: 'fastq'], [reads_1, reads_2]] } else { // If single end - [new_meta, reads_1] + [meta + [single_end: true, file_type: 'fastq'], reads_1] } } @@ -82,10 +84,10 @@ workflow READSUBMIT { study_accession_ch.first() ) - // Prepare input for submission with manifest and fastq files + // Prepare input for submission with manifest and read files submission_input = reads_ch.join(CREATE_READS_MANIFEST.out.manifest) - .map { meta, fastq, manifest -> - [meta, fastq, manifest] + .map { meta, reads, manifest -> + [meta, reads, manifest] } // --------- Upload data to ENA @@ -139,7 +141,18 @@ workflow READSUBMIT { : file("${projectDir}/assets/methods_description_template.yml", checkIfExists: true) def ch_methods_description = channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: true)) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{_meta, file -> file}) + // Split "ERX...,ERR..." accession column into experiment and run columns for the report + def ch_reads_accessions_split = CONCAT_ACCESSIONS.out.file_out + .flatMap { _meta, tsv -> tsv.readLines().drop(1).findAll { line -> line.trim() } } + .map { line -> + def (alias, accessions) = line.tokenize('\t') + [''] + def accession_list = accessions.tokenize(',')*.trim() + def experiment = accession_list.find { acc -> acc ==~ /^[EDS]RX\d+$/ } ?: '' + def run = accession_list.find { acc -> acc ==~ /^[EDS]RR\d+$/ } ?: '' + "${alias}\t${experiment}\t${run}" + } + .collectFile(name: 'reads_accessions_split.tsv', seed: "alias\texperiment\trun", newLine: true, sort: true) + ch_multiqc_files = ch_multiqc_files.mix(ch_reads_accessions_split) MULTIQC( ch_multiqc_files.flatten().collect().map { files -> [