Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ null/
.nf-test*
.idea/
test_data
.venv
7 changes: 6 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### `Fixed`

- [#116](https://github.com/nf-core/seqsubmit/pull/116)
- All tests except the `*_existing_study` ones now register a new submission study, so they can be run with any Webin account ([#95](https://github.com/nf-core/seqsubmit/issues/95)). The `test_mag_complete_metadata_existing_study` and `test_assembly_complete_metadata_existing_study` tests submit to the existing study `PRJEB98843` and require maintainer Webin credentials
- `test_upload = true` is set explicitly in all test configs
Comment thread
KateSakharova marked this conversation as resolved.
Resolved
- Study submission alias now has a random suffix, so parallel registrations started in the same second no longer collide
- `REGISTERSTUDY` now runs only when at least one input passed validation ([#113](https://github.com/nf-core/seqsubmit/issues/113))
- [#106](https://github.com/nf-core/seqsubmit/pull/106)
- Added support for metatranscriptome submission for ASSEMBLYSUBMIT
- Set retry to assembly_uploader
- Set retry to GENERATE_ASSEMBLY_MANIFEST module
- [#108](https://github.com/nf-core/seqsubmit/pull/108)
- Added test usecase in test-datasets https://github.com/nf-core/test-datasets/pull/2324 with contamination=0
- [#107](https://github.com/nf-core/seqsubmit/pull/107)
Expand Down
4 changes: 3 additions & 1 deletion bin/submit_study.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import logging
import os
import sys
import uuid
import xml.etree.ElementTree as ET
from io import BytesIO
from pathlib import Path
Expand Down Expand Up @@ -327,7 +328,8 @@ def build_submission_xml(
# SUBMISSION_SET
submission_set = ET.SubElement(webin, "SUBMISSION_SET")
submission = ET.SubElement(submission_set, "SUBMISSION")
sub_alias = f"study-submission-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
# Random suffix keeps aliases unique when several submissions start in the same second
sub_alias = f"study-submission-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:4]}"
submission.set("alias", sub_alias)
actions = ET.SubElement(submission, "ACTIONS")
main_action = ET.SubElement(actions, "ACTION")
Expand Down
5 changes: 3 additions & 2 deletions conf/test.config
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_multiple_bins_missing_metadata.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = params.pipelines_testdata_base_path + 'seqsubmit/test_data/small_cat_db/tax-db.tar.gz'
checkm2_db = null

}
6 changes: 4 additions & 2 deletions conf/test_assembly_complete_metadata.config
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,16 @@ process {

params {
config_profile_name = 'Test --mode metagenomic_assemblies complete_metadata profile'
config_profile_description = 'Tests single assembly submission with all metadata provided in the samplesheet so the pipeline skips coverage calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study.'
config_profile_description = 'Single-case assembly test with complete metadata values provided'

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/assembly_complete_metadata.csv'
outdir = 'test_output'

mode = "metagenomic_assemblies"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,15 @@ process {

params {
config_profile_name = 'Test --mode metagenomic_assemblies complete_metadata profile'
config_profile_description = 'Single-case assembly test with complete metadata values provided'
config_profile_description = 'Tests single assembly submission with all metadata provided in the samplesheet so the pipeline skips coverage calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study, so requires maintainer Webin credentials with access to PRJEB98843.'

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/assembly_complete_metadata.csv'
outdir = 'test_output'

mode = "metagenomic_assemblies"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
submission_study = "PRJEB98843"
centre_name = "TEST_CENTER"

test_upload = true

}
4 changes: 3 additions & 1 deletion conf/test_assembly_no_coverage_paired_reads.config
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ params {
outdir = 'test_output'

mode = "metagenomic_assemblies"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
}
4 changes: 3 additions & 1 deletion conf/test_assembly_no_coverage_single_reads.config
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ params {
outdir = 'test_output'

mode = "metagenomic_assemblies"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
}
4 changes: 3 additions & 1 deletion conf/test_assembly_one_contig.config
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ params {
outdir = 'test_output'

mode = "metagenomic_assemblies"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
}
4 changes: 3 additions & 1 deletion conf/test_full.config
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_multiple_bins_missing_metadata.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_complete_metadata.csv'
outdir = 'test_output'

mode = "mags"
submission_study = null
Expand All @@ -34,5 +35,4 @@ params {

cat_db = null
checkm2_db = null

}
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ process {

params {
config_profile_name = 'Test --mode mags complete_metadata profile'
config_profile_description = 'Tests single MAG submission with all metadata provided in the samplesheet, so the pipeline skips statistics and taxonomy calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study.'
config_profile_description = 'Tests single MAG submission with all metadata provided in the samplesheet, so the pipeline skips statistics and taxonomy calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study, so requires maintainer Webin credentials with access to PRJEB98843.'

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_complete_metadata.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
Comment thread
ochkalova marked this conversation as resolved.
Resolved
Expand All @@ -33,5 +34,4 @@ params {

cat_db = null
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_no_coverage_paired_reads.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_no_coverage_paired_reads.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = null
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_no_coverage_single_reads.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_no_coverage_single_reads.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = null
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_no_quality.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_no_quality.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
Expand All @@ -36,5 +38,4 @@ params {
// which is not ideal but necessary to test the pipeline's behaviour with missing quality values
// Track upstream issue here: https://github.com/chklovski/CheckM2/issues/140
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_no_rna_presence.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_no_rna_presence.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = null
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_no_taxonomy.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_no_taxonomy.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = params.pipelines_testdata_base_path + 'seqsubmit/test_data/small_cat_db/tax-db.tar.gz'
checkm2_db = null

}
5 changes: 3 additions & 2 deletions conf/test_mag_one_contig.conf
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ params {

// Input data
input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_one_contig.csv'
outdir = 'test_output'

mode = "mags"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true

cat_db = null
checkm2_db = null

}
3 changes: 2 additions & 1 deletion conf/test_reads_paired.config
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ params {
outdir = 'test_output'

mode = "reads"
submission_study = "PRJEB98843"
submission_study = null
study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json'
centre_name = "TEST_CENTER"

test_upload = true
Expand Down
2 changes: 1 addition & 1 deletion docs/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ All submissions are performed via [Webin-CLI](https://github.com/enasequence/web

### Testing Requirements

All new functionality must be covered by tests. Running nf-tests that perform ENA submission locally requires ENA webin credentials. Contributors may ask for testing credentials on the nf-core Slack [#seqsubmit](https://nfcore.slack.com/channels/seqsubmit) channel.
All new functionality must be covered by tests. Running nf-tests that perform ENA submission locally requires ENA Webin credentials. Tests that register a new study can be run with any free [Webin account](https://www.ebi.ac.uk/ena/submit/webin/login); this is every test except the `*_existing_study` ones. The `*_existing_study` tests submit to the existing study `PRJEB98843` and need maintainer credentials, which contributors may ask for on the nf-core Slack [#seqsubmit](https://nfcore.slack.com/channels/seqsubmit) channel.

At minimum, include:

Expand Down
32 changes: 16 additions & 16 deletions nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -239,22 +239,22 @@ profiles {
apptainer.runOptions = '--nv'
singularity.runOptions = '--nv'
}
test { includeConfig 'conf/test.config' }
test_full { includeConfig 'conf/test_full.config' }
test_mag_complete_metadata { includeConfig 'conf/test_mag_complete_metadata.conf' }
test_mag_no_study_complete_metadata { includeConfig 'conf/test_mag_no_study_complete_metadata.config' }
test_mag_no_coverage_single_reads { includeConfig 'conf/test_mag_no_coverage_single_reads.conf' }
test_mag_no_coverage_paired_reads { includeConfig 'conf/test_mag_no_coverage_paired_reads.conf' }
test_mag_no_quality { includeConfig 'conf/test_mag_no_quality.conf' }
test_mag_one_contig { includeConfig 'conf/test_mag_one_contig.conf' }
test_mag_no_rna_presence { includeConfig 'conf/test_mag_no_rna_presence.conf' }
test_mag_no_taxonomy { includeConfig 'conf/test_mag_no_taxonomy.conf' }
test_assembly_complete_metadata { includeConfig 'conf/test_assembly_complete_metadata.config' }
test_assembly_no_study_complete_metadata { includeConfig 'conf/test_assembly_no_study_complete_metadata.config' }
test_assembly_no_coverage_single_reads { includeConfig 'conf/test_assembly_no_coverage_single_reads.config' }
test_assembly_no_coverage_paired_reads { includeConfig 'conf/test_assembly_no_coverage_paired_reads.config' }
test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' }
test_reads_paired { includeConfig 'conf/test_reads_paired.config' }
test { includeConfig 'conf/test.config' }
test_full { includeConfig 'conf/test_full.config' }
test_mag_complete_metadata_existing_study { includeConfig 'conf/test_mag_complete_metadata_existing_study.conf' }
test_mag_complete_metadata { includeConfig 'conf/test_mag_complete_metadata.config' }
test_mag_no_coverage_single_reads { includeConfig 'conf/test_mag_no_coverage_single_reads.conf' }
test_mag_no_coverage_paired_reads { includeConfig 'conf/test_mag_no_coverage_paired_reads.conf' }
test_mag_no_quality { includeConfig 'conf/test_mag_no_quality.conf' }
test_mag_one_contig { includeConfig 'conf/test_mag_one_contig.conf' }
test_mag_no_rna_presence { includeConfig 'conf/test_mag_no_rna_presence.conf' }
test_mag_no_taxonomy { includeConfig 'conf/test_mag_no_taxonomy.conf' }
test_assembly_complete_metadata_existing_study { includeConfig 'conf/test_assembly_complete_metadata_existing_study.config' }
test_assembly_complete_metadata { includeConfig 'conf/test_assembly_complete_metadata.config' }
test_assembly_no_coverage_single_reads { includeConfig 'conf/test_assembly_no_coverage_single_reads.config' }
test_assembly_no_coverage_paired_reads { includeConfig 'conf/test_assembly_no_coverage_paired_reads.config' }
test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' }
test_reads_paired { includeConfig 'conf/test_reads_paired.config' }
}

// Load nf-core custom profiles from different institutions
Expand Down
5 changes: 4 additions & 1 deletion tests/assembly_complete_metadata.nf.test.snap
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"-profile test_assembly_complete_metadata": {
"content": [
7,
8,
{
"CONCAT_ACCESSIONS": {
"coreutils": 9.4,
Expand All @@ -22,6 +22,9 @@
"GENERATE_ASSEMBLY_MANIFEST": {
"assembly_uploader": "1.3.6"
},
"REGISTERSTUDY": {
"mgnify-pipelines-toolkit": "1.5.1"
},
"SUBMIT": {
"ena-webin-cli": "9.0.3",
"mgnify-pipelines-toolkit": "1.5.1",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
nextflow_pipeline {

name "Test assembly submission workflow - complete_metadata"
name "Test assembly submission workflow - complete_metadata (submission to existing study)"
script "../main.nf"
tag "pipeline"
tag "mode_assembly"
tag "test_assembly_no_study_complete_metadata"
profile "test_assembly_no_study_complete_metadata"
tag "test_assembly_complete_metadata_existing_study"
profile "test_assembly_complete_metadata_existing_study"

test("-profile test_assembly_no_study_complete_metadata") {
test("-profile test_assembly_complete_metadata_existing_study") {

when {
params {
Expand Down
Loading
Loading