Fix/update container bam

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### `Fixed`
 
+- [#117](https://github.com/nf-core/seqsubmit/pull/117) and [#119](https://github.com/nf-core/seqsubmit/pull/119)
+  - Added BAM files support as raw reads
+  - Added tests for BAM submission with `-submit` and `-validate` modes
+  - Added assigned `run accession` and `experiment accession` to MultiQC report ([Issue 70](https://github.com/nf-core/seqsubmit/issues/70) resolved)
 - [#116](https://github.com/nf-core/seqsubmit/pull/116)
   - All tests except the `*_existing_study` ones now register a new submission study, so they can be run with any Webin account ([#95](https://github.com/nf-core/seqsubmit/issues/95)). The `test_mag_complete_metadata_existing_study` and `test_assembly_complete_metadata_existing_study` tests submit to the existing study `PRJEB98843` and require maintainer Webin credentials
   - `test_upload = true` is set explicitly in all test configs

assets/multiqc_config.yml

@@ -21,6 +21,7 @@ custom_content:
     - sample_registration
     - submission_results_assemblies
     - submission_results_genomes
+    - submission_results_reads
 
 sp:
   assembly_metadata:
@@ -33,6 +34,8 @@ sp:
     fn_re: 'genomes_accessions\.tsv$'
   submission_results_assemblies:
     fn_re: 'assemblies_accessions\.tsv$'
+  submission_results_reads:
+    fn_re: 'reads_accessions_split\.tsv$'
 
 # Custom data configuration
 custom_data:
@@ -160,3 +163,26 @@ custom_data:
       accession:
         title: "ENA accession"
         description: "Assigned accession after submission"
+
+  submission_results_reads:
+    id: "submission_results_reads"
+    section_name: "Submission results"
+    description: >
+      Accessions assigned during upload process (experiment and run accessions).
+      Users should read the ENA documentation on referencing submitted data:
+      https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html
+    file_format: "tsv"
+    plot_type: "table"
+    pconfig:
+      id: "Reads assigned accessions"
+      title: "Submission results"
+    headers:
+      alias:
+        title: "alias"
+        description: "Read submission name"
+      experiment:
+        title: "ENA experiment"
+        description: "Experiment accession (ERX) assigned after submission"
+      run:
+        title: "ENA run"
+        description: "Run accession (ERR) assigned after submission"

assets/schema_input_reads.json

@@ -22,12 +22,21 @@
                 "meta": ["sample_accession"]
             },
             "fastq_1": {
-                "type": "string",
-                "format": "file-path",
-                "exists": true,
-                "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$",
+                "anyOf": [
+                    {
+                        "type": "string",
+                        "format": "file-path",
+                        "exists": true,
+                        "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$"
+                    },
+                    {
+                        "type": "string",
+                        "maxLength": 0
+                    }
+                ],
+                "default": null,
                 "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)",
-                "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped)"
+                "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Leave empty if reads are provided as BAM"
             },
             "fastq_2": {
                 "anyOf": [
@@ -46,6 +55,23 @@
                 "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)",
                 "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads"
             },
+            "bam": {
+                "anyOf": [
+                    {
+                        "type": "string",
+                        "format": "file-path",
+                        "exists": true,
+                        "pattern": "^\\S+\\.bam$"
+                    },
+                    {
+                        "type": "string",
+                        "maxLength": 0
+                    }
+                ],
+                "default": null,
+                "errorMessage": "BAM file must have extension '.bam'",
+                "description": "Path to reads in unaligned or aligned BAM format. Use instead of fastq_1/fastq_2"
+            },
             "platform": {
                 "type": "string",
                 "enum": [
@@ -223,12 +249,53 @@
         "required": [
             "id",
             "sample_accession",
-            "fastq_1",
             "platform",
             "instrument",
             "library_source",
             "library_selection",
             "library_strategy"
-        ]
+        ],
+        "oneOf": [
+            {
+                "properties": {
+                    "fastq_1": {
+                        "type": "string",
+                        "minLength": 1
+                    }
+                },
+                "required": ["fastq_1"]
+            },
+            {
+                "properties": {
+                    "bam": {
+                        "type": "string",
+                        "minLength": 1
+                    }
+                },
+                "required": ["bam"]
+            }
+        ],
+        "if": {
+            "properties": {
+                "bam": {
+                    "type": "string",
+                    "minLength": 1
+                }
+            },
+            "required": ["bam"]
+        },
+        "then": {
+            "properties": {
+                "fastq_1": {
+                    "maxLength": 0,
+                    "errorMessage": "fastq_1 must be empty when reads are provided as BAM"
+                },
+                "fastq_2": {
+                    "maxLength": 0,
+                    "errorMessage": "fastq_2 must be empty when reads are provided as BAM"
+                }
+            }
+        },
+        "errorMessage": "Reads must be provided either as FASTQ (fastq_1, optionally fastq_2) or as a single BAM (bam, with fastq_1 and fastq_2 empty)"
     }
 }

conf/modules.config

@@ -176,6 +176,12 @@ process {
         ]
     }
 
+    // Reads can be multi or 100s of GB (e.g. long-read BAMs): symlink instead of copying them before upload.
+    // assemblies, bins and mags modes keep the default stageInMode 'copy'.
+    withName: '.*:READSUBMIT:SUBMIT' {
+        stageInMode = 'symlink'
+    }
+
     withName: 'REGISTERSTUDY|CREATE_READS_MANIFEST' {
         publishDir = [
             enabled: false

conf/test_reads_long_bam.config

@@ -0,0 +1,36 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/seqsubmit -profile test_reads_long_bam,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+process {
+    resourceLimits = [
+        cpus: 2,
+        memory: '8.GB',
+        time: '1.h'
+    ]
+}
+
+params {
+    config_profile_name        = 'Test --mode reads with long reads in BAM format'
+    config_profile_description = 'Minimal test profile for submission of long reads in BAM format'
+
+    // Input data
+    input  = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv'
+    outdir = 'test_output'
+
+    mode             = "reads"
+    webincli_mode    = "submit"
+    submission_study = null
+    study_metadata   = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
+    centre_name      = "TEST_CENTER"
+
+    test_upload      = true
+}

conf/test_reads_long_bam_validate.config

@@ -0,0 +1,35 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/seqsubmit -profile test_reads_long_bam_validate,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+process {
+    resourceLimits = [
+        cpus: 2,
+        memory: '8.GB',
+        time: '1.h'
+    ]
+}
+
+params {
+    config_profile_name        = 'Test --mode reads with long reads in BAM format (validate only)'
+    config_profile_description = 'Minimal test profile for submission of long reads in BAM format (validation only)'
+
+    // Input data
+    input  = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_reads_bam.csv'
+    outdir = 'test_output'
+
+    mode             = "reads"
+    webincli_mode    = "validate"
+    submission_study = "PRJEB98843"
+    centre_name      = "TEST_CENTER"
+
+    test_upload      = true
+}

docs/output.md

@@ -67,7 +67,7 @@ When `--mode reads` is used, results are written under `reads/`.
 <summary>Output files</summary>
 
 - `reads/`
-  - `upload/reads_accessions.tsv`: run accessions assigned to submitted reads.
+  - `upload/reads_accessions.tsv`: experiment and run accessions assigned to submitted reads (also shown in the MultiQC report).
 
 </details>