diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c8ad298b..d3e7af90 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,14 +8,14 @@ These are the most common things requested on pull requests (PRs). Remember that PRs should be made against the dev branch, unless you're preparing a pipeline release. -Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/seqsubmit/tree/master/docs/CONTRIBUTING.md) +Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/seqsubmit/tree/main/docs/CONTRIBUTING.md) --> ## PR checklist - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! -- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/seqsubmit/tree/master/docs/CONTRIBUTING.md) +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/seqsubmit/tree/main/docs/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/seqsubmit _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core pipelines lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 35ddb800..cc6deee6 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -37,7 +37,7 @@ runs: - name: Setup apptainer if: contains(inputs.profile, 'singularity') - uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 # v2 + uses: eWaterCycle/setup-apptainer@4bb22c52d4f63406c49e94c804632975787312b3 # v2.0.0 - name: Set up Singularity if: contains(inputs.profile, 'singularity') diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml index 78d5dbe0..4974f44f 100644 --- a/.github/workflows/release-announcements.yml +++ b/.github/workflows/release-announcements.yml @@ -18,7 +18,7 @@ jobs: id: get_description run: | echo "description=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .description')" >> $GITHUB_OUTPUT - - uses: rzr/fediverse-action@563159eb8d45f70ab6aaba36ed55cd037e51f441 # master + - uses: rzr/fediverse-action@66c2cbb5b1997666b0e28d597631b6a4f09a2719 # v0.0.6 with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} host: "mstdn.science" # custom host if not "mastodon.social" (default) diff --git a/.nf-core.yml b/.nf-core.yml index 48469290..7c08c936 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -11,12 +11,11 @@ lint: - docs/images/nf-core-seqsubmit_logo_light.png - docs/images/nf-core-seqsubmit_logo_dark.png - assets/nf-core-seqsubmit_logo_light.png - - .github/workflows/branch.yml - - .github/workflows/linting.yml -nf_core_version: 4.0.2 +nf_core_version: 4.0.3 repository_type: pipeline template: - author: Martin Beracochea, Ekaterina Sakharova, Sofia Ochkalova, Evangelos Karatzas, Tim Rozday + author: Martin Beracochea, Ekaterina Sakharova, Sofia Ochkalova, Evangelos Karatzas, + Tim Rozday description: Submit sequences, assemblies, MAGs, etc., to public archives force: false is_nfcore: true diff --git a/CHANGELOG.md b/CHANGELOG.md index d12e8714..a716c4ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v1.0.0 - [date] +## v1.0.0 - 01/08/2026 Initial release of nf-core/seqsubmit, created with the [nf-core](https://nf-co.re/) template. diff --git a/README.md b/README.md index 40f2f6e6..ecf1b660 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.10.4-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/) -[![nf-core template version](https://img.shields.io/badge/nf--core_template-4.0.2-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/4.0.2) +[![nf-core template version](https://img.shields.io/badge/nf--core_template-4.0.3-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/4.0.3) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -21,244 +21,91 @@ ## Introduction +![seqsubmit workflow diagram](docs/images/seqsubmit_schema.png) + **nf-core/seqsubmit** is a Nextflow pipeline for submitting sequence data to [ENA](https://www.ebi.ac.uk/ena/browser/home). -Currently, the pipeline supports four submission modes, each routed to a dedicated workflow and requiring its own input samplesheet structure: +The pipeline currently supports the following submission modes, each routed to a dedicated workflow: + +- `reads` — raw sequencing reads submission via the `READSUBMIT` workflow (pink) +- `metagenomic_assemblies` — assembly submission via the `ASSEMBLYSUBMIT` workflow (green) +- `mags` — metagenome-assembled genomes (MAGs) submission via the `GENOMESUBMIT` workflow (blue) +- `bins` — bins submission via the `GENOMESUBMIT` workflow (blue) -- `mags` for Metagenome Assembled Genomes (MAGs) submission with `GENOMESUBMIT` workflow -- `bins` for bins submission with `GENOMESUBMIT` workflow -- `metagenomic_assemblies` for assembly submission with `ASSEMBLYSUBMIT` workflow -- `reads` for raw sequencing reads submission with `READSUBMIT` workflow + -![seqsubmit workflow diagram](assets/seqsubmit_schema.png) +Each workflow has its own samplesheet structure, prerequisites, and limitations — they are briefly described below. See the [usage documentation](https://nf-co.re/seqsubmit/usage) for more detailed explanations. ## Requirements - [Nextflow](https://www.nextflow.io/) `>=25.04.0` -- Webin account registered at https://www.ebi.ac.uk/ena/submit/webin/login -- Raw reads used to assemble contigs submitted to [INSDC](https://www.insdc.org/) and associated accessions available - -Setup your environment secrets before running the pipeline: - -`nextflow secrets set ENA_WEBIN "Webin-XXX"` - -`nextflow secrets set ENA_WEBIN_PASSWORD "XXX"` - -Make sure you update commands above with your authorised credentials. - -## Input samplesheets +- A Webin account registered at -For detailed descriptions of all samplesheet columns, see the [usage documentation](docs/usage.md#samplesheet-input). + Set your Webin credentials as Nextflow secrets: -### `mags` and `bins` modes (`GENOMESUBMIT`) + ```bash + nextflow secrets set ENA_WEBIN "Webin-XXX" + nextflow secrets set ENA_WEBIN_PASSWORD "XXX" + ``` -The input must follow `assets/schema_input_genome.json`. + Make sure to replace the values above with your own credentials. -Required columns: +- Provide either a study accession or a study registration metadata file for the study the submission will be associated with. See the [Submission study](docs/usage.md#submission-study) section of the usage documentation for details. -- `sample` -- `fasta` (must end with `.fa.gz`, `.fasta.gz`, or `.fna.gz`) -- `accession` -- `assembly_software` -- `binning_software` -- `binning_parameters` -- `metagenome` -- `environmental_medium` -- `broad_environment` -- `local_environment` -- `co-assembly` +- Depending on the chosen mode, samples, reads, or metagenomic assemblies must be pre-submitted to ENA to obtain the corresponding accessions, which you then reference in your submission. Refer to the relevant mode section in the [usage documentation](https://nf-co.re/seqsubmit/usage) for details. -At least one of the following must be provided per row: - -- reads (`fastq_1`, optional `fastq_2` for paired-end) -- `genome_coverage` - -Additional supported columns: - -- `stats_generation_software` -- `completeness` -- `contamination` -- `RNA_presence` -- `NCBI_lineage` - -If `genome_coverage`, `stats_generation_software`, `completeness`, `contamination`, `RNA_presence`, or `NCBI_lineage` are missing, the workflow can calculate or infer them when the required inputs are available. +## Input samplesheets -Those fields are metadata required for the [genome_uploader](https://github.com/EBI-Metagenomics/genome_uploader) package. +### `reads` mode -Example `samplesheet_genomes.csv`: +Example: ```csv -sample,fasta,accession,fastq_1,fastq_2,assembly_software,binning_software,binning_parameters,stats_generation_software,completeness,contamination,genome_coverage,metagenome,co-assembly,broad_environment,local_environment,environmental_medium,RNA_presence,NCBI_lineage -lachnospira_eligens,data/bin_lachnospira_eligens.fa.gz,SRR24458089,,,spades_v3.15.5,metabat2_v2.6,default,CheckM2_v1.0.1,61.0,0.21,32.07,sediment metagenome,No,marine,cable_bacteria,marine_sediment,No,d__Bacteria;p__Proteobacteria;s__unclassified_Proteobacteria +id,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description +illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ ``` -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. - -### `metagenomic_assemblies` mode (`ASSEMBLYSUBMIT`) +See the [`reads` mode section](docs/usage.md#samplesheet-input) of the usage documentation for more details. -The input must follow `assets/schema_input_assembly.json`. +### `metagenomic_assemblies` mode -Required columns: - -- `sample` -- `fasta` (must end with `.fa.gz`, `.fasta.gz`, or `.fna.gz`) -- `run_accession` -- `assembler` -- `assembler_version` - -At least one of the following must be provided per row: - -- reads (`fastq_1`, optional `fastq_2` for paired-end) -- `coverage` - -If `coverage` is missing and reads are provided, the workflow calculates average coverage with `coverm`. - -Example `samplesheet_assembly.csv`: +Example: ```csv -sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version assembly_1,data/contigs_1.fasta.gz,data/reads_1.fastq.gz,data/reads_2.fastq.gz,,ERR011322,SPAdes,3.15.5 assembly_2,data/contigs_2.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9 ``` -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. - -### `reads` mode (`READSUBMIT`) - -The input must follow `assets/schema_input_reads.json`. +See the [`metagenomic_assemblies` mode section](docs/usage.md#samplesheet-input-1) of the usage documentation for more details. -Required columns: +### `mags` and `bins` modes -- `sample` -- `sample_accession` -- `fastq_1` -- `fastq_2` -- `platform` -- `instrument` -- `library_source` -- `library_selection` -- `library_strategy` - -Optional columns: - -- `insert_size` -- `library_name` -- `description` - -Example `samplesheet_reads.csv`: +Example: ```csv -sample,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description -illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ +id,fasta,accession,fastq_1,fastq_2,assembly_software,binning_software,binning_parameters,stats_generation_software,completeness,contamination,genome_coverage,metagenome,co-assembly,broad_environment,local_environment,environmental_medium,RNA_presence,NCBI_lineage +lachnospira_eligens,data/bin_lachnospira_eligens.fa.gz,SRR24458089,,,spades_v3.15.5,metabat2_v2.6,default,CheckM2_v1.0.1,61.0,0.21,32.07,sediment metagenome,false,marine,cable_bacteria,marine_sediment,false,d__Bacteria;p__Proteobacteria;s__unclassified_Proteobacteria ``` -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. +See the [`mags` and `bins` modes section](docs/usage.md#samplesheet-input-2) of the usage documentation for the full list of required and optional columns. ## Usage > [!NOTE] > If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/get_started/environment_setup/overview) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/get_started/run-your-first-pipeline) with `-profile test` before running the workflow on actual data. -### Submission study - -All data submitted through this pipeline must be associated with an ENA study (project). You can either pass an accession of your existing study via `--submission_study`or provide a metadata file via `--study_metadata` and the pipeline will register the study with ENA before submitting your data. - -See the [usage documentation](docs/usage.md#submission-study) for more details. - -### Data privacy +### Running the pipeline -You can reference private ENA data if you have access to it via your Webin account (it was submitted previously under your credentials). To use private data accessions in your submission metadata, specify the `--is_private` flag. The pipeline will then use your provided Webin credentials to fetch the required metadata. - -You can also control the privacy of your submitted data: - -- If you provide an existing study via `--submission_study`, your submission will inherit the same privacy status as that study. -- If no study is provided, the pipeline will register a new one for you. To keep this new study private, you must specify a release date using `--release_date YYYY-MM-DD` (up to 2 years from the current date). - -| Example | Source Data | Submission Visibility | Required Arguments | Result | -| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------- | ------------------------------------------------------ | ----------------------------------------------------- | -| Assemblies generated from public reads. Assemblies should be public immediately after submission | public | public | – | SUCCESS | -| Assemblies generated from public reads. Assemblies should remain private for 1 year after submission | public | private | `--release_date YYYY-MM-DD` (+1 year) | SUCCESS | -| Bins generated from private reads and assemblies. Bins should be public after submission. User **has access** to the reads and assemblies via Webin account | private | public | `--is_private` | SUCCESS | -| Bins generated from private reads and assemblies. Bins should remain private for 2 years after submission. User **has access** via Webin account | private | private | `--is_private`, `--release_date YYYY-MM-DD` (+2 years) | SUCCESS | -| MAGs generated from private reads and assemblies. MAGs should be public after submission. User **does not have access** to the reads and assemblies via Webin account | private | public | – | ERROR (submission can only reference accessible data) | - -### Database setup (`CheckM2` and `CAT_pack`) - -The `mags`/`bins` workflow requires databases for completeness/contamination estimation and taxonomy assignment. See [Usage documentation](usage.md) for details. - -### Required parameters: - -| Parameter | Description | -| ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | -| `--mode` | Type of the data to be submitted. Options: `[mags, bins, metagenomic_assemblies, reads]` | -| `--input` | Path to the samplesheet describing the data to be submitted | -| `--outdir` | Path to the output directory for pipeline results | -| `--submission_study` OR `--study_metadata` | ENA study accession (PRJ/ERP) to submit the data to OR metadata file in JSON/TSV/CSV format to register new study | -| `--centre_name` | Name of the submitter's organisation | - -### Optional parameters: - -| Parameter | Description | -| ----------------- | ------------------------------------------------------------------------------------------ | -| `--upload_tpa` | Flag to control the type of assembly study (third party assembly or not). Default: false | -| `--test_upload` | Upload to TEST ENA server instead of LIVE. Default: true | -| `--webincli_mode` | Choose Webin-CLI mode: `submit` or `validate`. Default: `submit` | -| `--private` | Use that flag if you are referring to private data in ENA | -| `--release_date` | Use that flag if you want to keep your data private after submission until particular date | - -General command template: - -```bash -nextflow run nf-core/seqsubmit \ - -profile \ - --mode \ - --input \ - --centre_name \ - --submission_study \ - --outdir -``` - -Test run (submission to the ENA TEST server) in `mags` mode: - -```bash -nextflow run nf-core/seqsubmit \ - -profile docker \ - --mode mags \ - --input assets/samplesheet_genomes.csv \ - --submission_study \ - --centre_name TEST_CENTER \ - --webincli_mode submit \ - --test_upload true \ - --outdir results/validate_mags -``` - -Test run (submission to the ENA TEST server) in `metagenomic_assemblies` mode: - -```bash -nextflow run nf-core/seqsubmit \ - -profile docker \ - --mode metagenomic_assemblies \ - --input assets/samplesheet_assembly.csv \ - --submission_study \ - --centre_name TEST_CENTER \ - --webincli_mode submit \ - --test_upload true \ - --outdir results/validate_assemblies -``` - -Live submission example: +Each mode also has its own additional parameters and example commands — see the [usage documentation](https://nf-co.re/seqsubmit/usage) for details. General command template: ```bash nextflow run nf-core/seqsubmit \ - -profile docker \ - --mode metagenomic_assemblies \ - --input assets/samplesheet_assembly.csv \ - --submission_study PRJEB98843 \ - --test_upload false \ - --webincli_mode submit \ - --outdir results/live_assembly + -profile \ + --mode \ + --input \ + --centre_name \ + --submission_study \ + --outdir ``` > [!WARNING] @@ -270,8 +117,9 @@ For more details and further functionality, please refer to the [usage documenta Key output locations in `--outdir`: -- `mags/` or `bins/`: genome metadata, manifests, and per-sample submission support files +- `reads/`: per-sample submission receipts and accessions - `metagenomic_assemblies/`: assembly metadata CSVs and per-sample coverage files +- `mags/` or `bins/`: genome metadata, manifests, and per-sample submission support files - `multiqc/`: MultiQC summary report - `pipeline_info/`: execution reports, trace, DAG, and software versions @@ -279,7 +127,10 @@ For full details, see the [output documentation](https://nf-co.re/seqsubmit/outp ## Credits -nf-core/seqsubmit was originally written by [Martin Beracochea](https://github.com/mberacochea), [Ekaterina Sakharova](https://github.com/KateSakharova), [Sofia Ochkalova](https://github.com/ochkalova), [Evangelos Karatzas](https://github.com/vagkaratzas) and [Tim Rozday](https://github.com/timrozday-mgnify). +nf-core/seqsubmit was originally written by [Martin Beracochea](https://github.com/mberacochea), [Ekaterina Sakharova](https://github.com/KateSakharova), [Sofia Ochkalova](https://github.com/ochkalova) and [Tim Rozday](https://github.com/timrozday-mgnify). + +We thank the following people for their extensive assistance in the development of this pipeline: +[Evangelos Karatzas](https://github.com/vagkaratzas) ## Contributions and Support diff --git a/assets/schema_input_assembly.json b/assets/schema_input_assembly.json index 2b5fcea1..f6a85936 100644 --- a/assets/schema_input_assembly.json +++ b/assets/schema_input_assembly.json @@ -7,10 +7,11 @@ "items": { "type": "object", "properties": { - "sample": { + "id": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample must be provided and cannot contain spaces", + "description": "A unique identifier for this data entry. Must be globally unique within the input dataset", + "errorMessage": "ID must be provided and cannot contain spaces", "meta": ["id"] }, "fasta": { @@ -19,7 +20,7 @@ "exists": true, "pattern": "^([\\S\\s]*\\/)?[^\\s\\/]+\\.(fa|fasta|fna)\\.gz$", "errorMessage": "FASTA file must be provided and have extension '.fa.gz', '.fasta.gz', '.fna.gz'", - "description": "Metagenomic assembly FASTA file" + "description": "Path to assembly contigs in FASTA format compressed with gzip" }, "fastq_1": { "anyOf": [ @@ -34,8 +35,9 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Forward reads if paired-end or single-end reads FASTQ file" + "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required if coverage is not provided" }, "fastq_2": { "anyOf": [ @@ -50,43 +52,49 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Reverse reads FASTQ file if paired-end. Leave empty for single-end reads" + "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads" }, "coverage": { "anyOf": [ { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, { "type": "string", "maxLength": 0 } ], + "default": null, "errorMessage": "Coverage must be a positive number or empty", - "description": "Estimated value of assembly coverage" + "description": "Estimated sequencing depth of the assembly (optional). If this value is missing, it is computed automatically during pipeline execution when reads are provided", + "meta": ["coverage"] }, "run_accession": { "type": "string", "pattern": "^\\S+$", "errorMessage": "Accession must be provided and cannot contain spaces", - "description": "Accession of the run used to generate the assembly" + "description": "ENA run accession for the reads used to generate the metagenomic assembly. Reads must already be submitted to ENA", + "meta": ["run_accession"] }, "assembler": { "type": "string", "pattern": "^\\S+$", "errorMessage": "Assembler must be provided and cannot contain spaces", - "description": "Name of the assembler software used to generate the assembly" + "description": "Name of the assembler software used to generate the assembly, e.g. MEGAHIT, SPAdes, metaSPAdes, Flye, etc", + "meta": ["assembler"] }, "assembler_version": { "anyOf": [{ "type": "string" }, { "type": "number" }], "pattern": "^\\S+$", "errorMessage": "Assembler version must be provided and cannot contain spaces", - "description": "Version of the assembler software used to generate the assembly" + "description": "Version of the assembler software used to generate the assembly, e.g. 1.0.0", + "meta": ["assembler_version"] } }, - "required": ["sample", "fasta", "run_accession", "assembler", "assembler_version"], + "required": ["id", "fasta", "run_accession", "assembler", "assembler_version"], "anyOf": [ { "properties": { @@ -101,7 +109,7 @@ "properties": { "coverage": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 } }, "required": ["coverage"] diff --git a/assets/schema_input_genome.json b/assets/schema_input_genome.json index f851f856..4a012224 100644 --- a/assets/schema_input_genome.json +++ b/assets/schema_input_genome.json @@ -7,10 +7,11 @@ "items": { "type": "object", "properties": { - "sample": { + "id": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces", + "description": "A unique identifier for this data entry. Must be globally unique within the input dataset", + "errorMessage": "ID must be provided and cannot contain spaces", "meta": ["id"] }, "fasta": { @@ -18,12 +19,13 @@ "format": "file-path", "exists": true, "pattern": "^([\\S\\s]*\\/)?[^\\s\\/]+\\.(fa|fasta|fna)\\.gz$", - "errorMessage": "FASTA file for sequences 1 must be provided, cannot contain spaces and must have extension '.fa.gz', '.fasta.gz', or '.fna.gz'", - "description": "MAG/bin sequence file" + "errorMessage": "Path to the FASTA file must be provided, cannot contain spaces and must have extension '.fa.gz', '.fasta.gz', or '.fna.gz'", + "description": "Path to MAG/bin sequences in FASTA format compressed with `gzip`. All names of the FASTA files must be unique to prevent pipeline errors. File should contain at least one sequence" }, "accession": { "type": "string", - "description": "Run or assembly ENA accession" + "description": "ENA accession of the run or metagenomic assembly used to generate the MAG/bin. Reads or assembly must already be submitted to ENA", + "meta": ["accession"] }, "fastq_1": { "anyOf": [ @@ -38,8 +40,9 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Forward reads if paired-end or single-end reads FASTQ file" + "description": "Path to single-end/paired-end forward reads (in FASTQ format, optionally gzipped) used to generate the source metagenomic assembly. Required if genome_coverage is not provided" }, "fastq_2": { "anyOf": [ @@ -54,68 +57,126 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Reverse reads FASTQ file if paired-end. Leave empty for single-end reads" + "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped) used to generate the source metagenomic assembly. Leave empty for single-end reads" }, "assembly_software": { "type": "string", - "description": "Tool name and version that was used to assemble data" + "description": "Tool name and version that was used to assemble data, e.g. MEGAHIT_v1.0, SPAdes_v4.0.0, metaSPAdes_v3.15.0", + "meta": ["assembly_software"] }, "binning_software": { "type": "string", - "description": "Tool name and version that was used to bin data" + "description": "Tool name and version that was used to bin data, e.g. MetaBAT2_v1.0, VAMB_2.0, MaxBin2_v1, CONCOCT_v3.0, SemiBin2_v1, COMEBin_v2.3.4, etc", + "meta": ["binning_software"] }, "binning_parameters": { "type": "string", - "description": "Arguments used to bin data different from default" + "description": "Arguments used to bin data different from default, e.g. -min_contig_length 1000, otherwise use 'default'", + "meta": ["binning_parameters"] }, "stats_generation_software": { "type": "string", - "description": "Tool(s) used to estimate completeness and contamination" + "default": null, + "description": "Tool, including version, that was used to calculate completeness and contamination, e.g. CheckM2_v1.0.0", + "meta": ["stats_generation_software"] }, "completeness": { - "type": "number", - "description": "MAG/bin completeness value" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "description": "MAG/bin completeness score: the 0<= ratio <= 100 of observed single-copy marker genes to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined", + "meta": ["completeness"] }, "contamination": { - "type": "number", - "description": "MAG/bin contamination value" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "description": "MAG/bin contamination score: the 0<= ratio <= 100 of observed single-copy marker genes in ≥2 copies to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined", + "meta": ["contamination"] }, "genome_coverage": { - "type": "number", - "description": "MAG/bin coverage value" + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "description": "Estimated average sequencing depth (>0) across the genome. If the value is missing, it is computed automatically during pipeline execution when reads are provided", + "meta": ["genome_coverage"] }, "metagenome": { "type": "string", - "description": "ENA accepted metagenome name" + "description": "Registered metagenome taxonomic identifier or name that matches an existing ENA taxonomy entry. It needs to be listed in the taxonomy tree https://www.ebi.ac.uk/ena/browser/view/408169?show=tax-tree (you might need to press \"Tax tree - Show\" in the right most section of the page). Full list can also be found https://github.com/EBI-Metagenomics/genome_uploader/blob/main/genomeuploader/constants.py#L22", + "meta": ["metagenome"] }, "co-assembly": { - "type": "string", - "description": "Yes if data was co-assembled, No otherwise" + "type": "boolean", + "description": "Whether a co-assembly strategy was used for the initial metagenomic assembly generation. Options: 'true' or 'false'", + "meta": ["co_assembly"] }, "broad_environment": { "type": "string", - "description": "broad ecological context of a sample" + "description": "Broad ecological context of the sample, for example 'marine biome', 'desert biome'. It is recommended to use subclasses of EnvO 'biome' class (http://purl.obolibrary.org/obo/ENVO_00000428). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS", + "meta": ["broad_environment"] }, "local_environment": { "type": "string", - "description": "local ecological context" + "description": "Local environmental context of the sample, for example 'tropical dry broadleaf forest biome', 'marine abyssal zone biome'. It is recommended to use EnvO terms which are of smaller spatial grain than your entry for \"broad-scale environmental context\". Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS", + "meta": ["local_environment"] }, "environmental_medium": { "type": "string", - "description": "material displaced by the sample" + "description": "Material surrounding and in contact with the sample, for example 'mucus', 'lake water'. It is recommended to use subclasses of EnvO 'environmental material' class (http://purl.obolibrary.org/obo/ENVO_00010483). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS", + "meta": ["environmental_medium"] }, "RNA_presence": { - "type": "string", - "description": "Yes/No if at least 18 tRNAs and 5S, 16S, 23S rRNA genes detected" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "description": "Presence or absence of the 23S, 16S, and 5S rRNA genes and at least 18 tRNAs. This is used for MISAG/MIMAG assembly quality classification. Options: 'true' or 'false'", + "meta": ["RNA_presence"] }, "NCBI_lineage": { "type": "string", - "description": "full NCBI lineage - format: x;y;z" + "default": null, + "description": "Full NCBI lineage - format: x;y;z. For example, the lineage for E. coli can be: 'Bacteria;Pseudomonadati;Pseudomonadota;Gammaproteobacteria;Enterobacterales;Enterobacteriaceae;Escherichia' or '2;1224;1236;91347;543;561;562'. For more info check https://www.ncbi.nlm.nih.gov/datasets/docs/v2/data-processing/taxonomy-processing/taxonomy/", + "meta": ["NCBI_lineage"] } }, "required": [ - "sample", + "id", "fasta", "accession", "assembly_software", @@ -141,7 +202,7 @@ "properties": { "genome_coverage": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 } }, "required": ["genome_coverage"] diff --git a/assets/schema_input_reads.json b/assets/schema_input_reads.json index 25dfb318..c511c3cc 100644 --- a/assets/schema_input_reads.json +++ b/assets/schema_input_reads.json @@ -7,18 +7,19 @@ "items": { "type": "object", "properties": { - "sample": { + "id": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample must be provided and cannot contain spaces", + "errorMessage": "ID must be provided and cannot contain spaces", "meta": ["id"], - "description": "Unique experiment/run name" + "description": "Unique identifier of this particular data entry. Used as an experiment name" }, "sample_accession": { "type": "string", "pattern": "^\\S+$", "errorMessage": "Sample accession must be provided and cannot contain spaces", - "description": "ENA sample accession of the sample used to generate the reads" + "description": "ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA", + "meta": ["sample_accession"] }, "fastq_1": { "type": "string", @@ -26,7 +27,7 @@ "exists": true, "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$", "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Forward reads FASTQ file (single-end or paired-end)" + "description": "Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped)" }, "fastq_2": { "anyOf": [ @@ -41,8 +42,9 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Reverse reads FASTQ file if paired-end. Leave empty for single-end reads" + "description": "Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads" }, "platform": { "type": "string", @@ -63,13 +65,15 @@ "ULTIMA", "VELA_DIAGNOSTICS" ], - "description": "Sequencing platform. Must be one of the ENA controlled vocabulary values listed in the enum." + "description": "Sequencing platform. Must be one of the ENA controlled vocabulary values listed in https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation.", + "meta": ["platform"] }, "instrument": { "type": "string", "pattern": "^[^\\n]+$", "errorMessage": "Instrument must be provided and cannot span multiple lines", - "description": "Sequencer model (e.g., 'Illumina HiSeq 2000', 'PacBio Sequel')" + "description": "Sequencer model (e.g., 'Illumina HiSeq 2000', 'PacBio Sequel'). Must be one of the allowed ENA controlled vocabulary values listed in https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument", + "meta": ["instrument"] }, "library_source": { "type": "string", @@ -84,7 +88,8 @@ "VIRAL RNA", "OTHER" ], - "description": "Library source. Must be one of the ENA controlled vocabulary values listed in the enum." + "description": "Library source type. Must be one of the allowed ENA controlled vocabulary values listed in https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source", + "meta": ["library_source"] }, "library_selection": { "type": "string", @@ -121,7 +126,8 @@ "other", "unspecified" ], - "description": "Library selection. Must be one of the ENA controlled vocabulary values listed in the enum." + "description": "Library selection. Must be one of the allowed ENA controlled vocabulary values listed in https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection", + "meta": ["library_selection"] }, "library_strategy": { "type": "string", @@ -168,7 +174,8 @@ "Tethered Chromatin Conformation Capture", "OTHER" ], - "description": "Library strategy. Must be one of the ENA controlled vocabulary values listed in the enum." + "description": "Library strategy. Must be one of the ENA controlled vocabulary values listed in https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy", + "meta": ["library_strategy"] }, "insert_size": { "anyOf": [ @@ -181,8 +188,10 @@ "maxLength": 0 } ], + "default": null, "errorMessage": "Insert size must be a positive number or empty", - "description": "Fragment/insert size for paired-end reads (optional)" + "description": "Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Skip if not applicable.", + "meta": ["insert_size"] }, "library_name": { "anyOf": [ @@ -194,7 +203,8 @@ "maxLength": 0 } ], - "description": "Descriptive library name (optional)" + "description": "Descriptive library name (optional)", + "meta": ["library_name"] }, "description": { "anyOf": [ @@ -206,11 +216,12 @@ "maxLength": 0 } ], - "description": "Free-text description of the experiment (optional)" + "description": "Free-text description of the experiment (optional)", + "meta": ["description"] } }, "required": [ - "sample", + "id", "sample_accession", "fastq_1", "platform", diff --git a/assets/seqsubmit_schema.png b/assets/seqsubmit_schema.png deleted file mode 100644 index 475bdf3c..00000000 Binary files a/assets/seqsubmit_schema.png and /dev/null differ diff --git a/bin/count_rna.py b/bin/count_rna.py index a514d37f..8f340118 100755 --- a/bin/count_rna.py +++ b/bin/count_rna.py @@ -151,7 +151,7 @@ def main(): logging.info(f"Found {rrna_count} rRNA subunit types in total") # rrna_count should be 3 to make sure all 3 rRNA genes are present - decision = "Yes" if trna_count >= args.trna_limit and rrna_count == 3 else "No" + decision = "True" if trna_count >= args.trna_limit and rrna_count == 3 else "False" logging.info(f"RNA presented: {decision}") with open(args.output, 'w') as file_out: diff --git a/bin/submit_study.py b/bin/submit_study.py index 23f43fd5..6c8b0d49 100755 --- a/bin/submit_study.py +++ b/bin/submit_study.py @@ -379,7 +379,6 @@ def _add_project_element( sp = ET.SubElement(project, "SUBMISSION_PROJECT") ET.SubElement(sp, "SEQUENCING_PROJECT") - # TODO: Check existing_study_type and new_study_type metadata fields, do we need those? study_type = study.get("existing_study_type") if study_type: attrs = ET.SubElement( diff --git a/conf/containers_docker_arm64.config b/conf/containers_docker_arm64.config index 369f743a..11a7ab95 100644 --- a/conf/containers_docker_arm64.config +++ b/conf/containers_docker_arm64.config @@ -1 +1,2 @@ +process { withName: 'FALINT' { container = 'community.wave.seqera.io/library/fa-lint:1.2.0--9dec293ee366e2bd' } } process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.35--5c84a5000a226ab5' } } diff --git a/conf/test_assembly_complete_metadata.config b/conf/test_assembly_complete_metadata.config index 16e4ccf0..801cab7b 100644 --- a/conf/test_assembly_complete_metadata.config +++ b/conf/test_assembly_complete_metadata.config @@ -20,7 +20,7 @@ process { params { config_profile_name = 'Test --mode metagenomic_assemblies complete_metadata profile' - config_profile_description = 'Single-case assembly test with complete metadata values provided' + config_profile_description = 'Tests single assembly submission with all metadata provided in the samplesheet so the pipeline skips coverage calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study.' // Input data input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/assembly_complete_metadata.csv' diff --git a/conf/test_mag_complete_metadata.conf b/conf/test_mag_complete_metadata.conf index a25e31b7..428878a3 100644 --- a/conf/test_mag_complete_metadata.conf +++ b/conf/test_mag_complete_metadata.conf @@ -20,7 +20,7 @@ process { params { config_profile_name = 'Test --mode mags complete_metadata profile' - config_profile_description = 'Single-case MAG test with complete metadata values provided' + config_profile_description = 'Tests single MAG submission with all metadata provided in the samplesheet, so the pipeline skips statistics and taxonomy calculation and only performs the upload to ENA. Submits to a pre-existing ENA study passed via --submission_study.' // Input data input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/mag_complete_metadata.csv' diff --git a/conf/test_mag_no_quality.conf b/conf/test_mag_no_quality.conf index 9d3724ab..656bffea 100644 --- a/conf/test_mag_no_quality.conf +++ b/conf/test_mag_no_quality.conf @@ -34,6 +34,7 @@ params { cat_db = null // CheckM2 doesn't allow usage of small test database, so real database will be downloaded during the test run, // which is not ideal but necessary to test the pipeline's behaviour with missing quality values + // Track upstream issue here: https://github.com/chklovski/CheckM2/issues/140 checkm2_db = null } diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 67a43704..7330210d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -184,44 +184,26 @@ If you update images or graphics, follow the nf-core [style guidelines](https:// To ensure consistency and maintainability of the nf-core/seqsubmit pipeline, contributors should follow these guidelines when implementing new features or modifying existing functionality. -1. Understand Workflow Structure +### Code style -The pipeline consists of independent workflows, each corresponding to a certain data type: +The pipeline is organized into three separate workflows, one for each mode (mags, bins, metagenomic_assemblies, reads), because each data type has distinct submission procedures and metadata requirements. Future development should follow the same structure to maintain consistency and readability. -- GENOMESUBMIT (mags, bins) -- ASSEMBLYSUBMIT (metagenomic_assemblies) -- READSUBMIT (raw reads) +If a new feature extends support for an **existing** data type (for example, _adding support for eukaryotic MAGs_), the corresponding logic should be integrated into the existing workflow rather than creating a new one. -When adding new functionality (e.g. a new submission mode such as `isolate_genomes`), ensure that: +To improve readability, divide the main workflow into logical sections using comments that begin with `// ---------`, for example: -- It integrates cleanly into the appropriate workflow or introduces a well-structured new one. -- Existing workflows remain unaffected unless changes are explicitly required. -- Shared logic is abstracted into reusable modules or subworkflows where possible. - -⸻ - -2. Follow Samplesheet and Schema-Driven Design - -All inputs in seqsubmit are defined via strict samplesheet schemas. - -- Any new feature must: - - Extend or introduce a corresponding schema (assets/schema*input*\*.json) - - Clearly define required and optional fields -- Maintain: - - Column order consistency (critical for pipeline execution) - - Backward compatibility where possible -- Avoid duplicating metadata logic—reuse existing parsing and validation patterns. +``` +// --------- Upload data to ENA +``` -⸻ +Use standard `//` comments for shorter explanations within each section. -3. ENA Submission Requirements +### ENA Submission Requirements Submissions to [ENA](https://www.ebi.ac.uk/ena/browser/home) are governed by a well-defined data model and submission process. These include relationships between entities (e.g. studies, samples, experiments, runs, analyses), required submission steps, and accepted file types and formats. Contributors should refer directly to the [official ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/general-guide.html) to understand latest submission requirements and ensure that pipeline behaviour remains aligned with ENA standards. - Follow ENA-defined submission workflows and required steps for each data type - - Ensure that: - New pipeline workflows follow ENA-defined submission workflows and required steps for each data type - Metadata fields align with ENA requirements @@ -232,13 +214,13 @@ Contributors should refer directly to the [official ENA documentation](https://e ⸻ -4. Integration with Webin-CLI +### Integration with Webin-CLI All submissions are performed via [Webin-CLI](https://github.com/enasequence/webin-cli). Contributors should rely on official Webin-CLI documentation to understand expected inputs, parameters, and modes of operation. Make sure to always use the latest version of the tool. - Contributions affecting submission logic must: - - Respect --webincli_mode (validate vs submit) - - Support --test_upload for safe testing against ENA test server + - Respect `--webincli_mode` (validate vs submit) + - Support `--test_upload` for safe testing against ENA test server - Never introduce changes that: - Risk accidental submission to production without explicit user intent - Ensure compatibility with: @@ -246,9 +228,9 @@ All submissions are performed via [Webin-CLI](https://github.com/enasequence/web ⸻ -5. Testing Requirements +### Testing Requirements -All new functionality must be covered by tests. +All new functionality must be covered by tests. Running nf-tests that perform ENA submission locally requires ENA webin credentials. Contributors may ask for testing credentials on the nf-core Slack [#seqsubmit](https://nfcore.slack.com/channels/seqsubmit) channel. At minimum, include: @@ -268,36 +250,12 @@ Where applicable: ⸻ -6. Performance and Resource Awareness - -Some data processing steps may be resource-intensive (e.g. database preparation for CAT_pack). When introducing new workflows or features avoid unnecessary recomputation. Add option to reuse cached results or provide prebuilt databases when possible. - -⸻ - -7. Documentation and Usability - -Every contribution should improve usability: - -- Update: - - README.md - - docs/usage.md - - Schema descriptions -- Provide: - - Clear examples (e.g. samplesheets) - - Meaningful parameter descriptions - -The goal is to ensure that users can successfully submit data without needing deep knowledge of ENA submission requirements. - -⸻ - -8. Design Principles +### Performance and Resource Awareness -When contributing, prioritise: +Some data processing steps may be resource-intensive (e.g. database preparation for CAT_pack). -- Reproducibility (consistent results across environments) -- Automation (minimise manual user intervention) -- Clarity (explicit inputs and outputs) -- Extensibility (easy to add new submission types) +- When introducing new workflows or features avoid unnecessary recomputation. +- Add option to reuse cached results or provide prebuilt databases when possible. ⸻ diff --git a/docs/images/seqsubmit_schema.png b/docs/images/seqsubmit_schema.png new file mode 100644 index 00000000..df619e5a Binary files /dev/null and b/docs/images/seqsubmit_schema.png differ diff --git a/docs/images/seqsubmit_schema.svg b/docs/images/seqsubmit_schema.svg new file mode 100644 index 00000000..53521eb4 --- /dev/null +++ b/docs/images/seqsubmit_schema.svg @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/methods.md b/docs/methods.md index 0f15a335..1504fa1e 100644 --- a/docs/methods.md +++ b/docs/methods.md @@ -1,15 +1,40 @@ # nf-core/seqsubmit: Methods -## Overview +This page provides a detailed overview of the methods and procedures that are executed throughout the nf-core/seqsubmit pipeline. -`nf-core/seqsubmit` currently contains two workflow implementations: +## `ASSEMBLYSUBMIT` -- `GENOMESUBMIT` for `--mode mags` and `--mode bins` -- `ASSEMBLYSUBMIT` for `--mode metagenomic_assemblies` +### Overview + +The `ASSEMBLYSUBMIT` workflow: + +1. Reads the samplesheet and associated assembly FASTA files. +2. Validates assembly FASTA files. +3. Reuses provided or calculates missing `coverage` values. +4. Builds the metadata table used for manifest generation. +5. Generates assembly manifests and submits the assemblies to ENA. + +### Assembly FASTA validation + +Assembly FASTA files are validated with `fa-lint` before downstream processing. Each file is checked for FASTA format validity and contig count. A genome must contain at least two contigs to pass validation, which is an ENA requirement for contig-level submissions. -This page documents the methods that are currently implemented in the pipeline. +Only assemblies with successful validation are forwarded to coverage estimation, metadata/manifest generation and submission. -## `GENOMESUBMIT` methods +### Coverage calculation + +If the `coverage` column is already populated in the samplesheet, that value is used directly. + +If `coverage` is missing, the workflow calculates coverage with `coverm contig`. + +`coverm contig` outputs per-contig depth. The workflow then reads this file and calculates the arithmetic mean across all contigs. + +If the per-contig coverage values are $c_1, c_2, \ldots, c_n$, the workflow currently computes assembly coverage as an unweighted mean across contigs: + +$$ +\bar{c} = \frac{1}{n} \sum_{i=1}^{n} c_i +$$ + +## `GENOMESUBMIT` ### Overview @@ -17,22 +42,24 @@ The `GENOMESUBMIT` workflow: 1. Reads the samplesheet and associated genome FASTA files. 2. Validates genome FASTA files. -3. Reuses provided or calculates missing values for RNA genes presence, coverage, taxonomy, and genome quality metrics. +3. Reuses provided or calculates missing values for tRNA and rRNA genes presence, coverage, taxonomy, and genome quality metrics. 4. Collects genome metadata into the tabular format required by `genome_uploader`. 5. Generates submission manifests for ENA. 6. Performs submission to ENA. ### Genome FASTA validation -Genome FASTA files are validated with the `FASTAVALIDATOR` module before downstream processing. Each file is checked for FASTA format validity and contig count. A genome must contain at least two contigs to pass validation, which is an ENA requirement for contig-level submissions. +Genome FASTA files are validated with the `fa-lint` before downstream processing. Each file is checked for FASTA format validity and contig count. A genome must contain at least two contigs to pass validation, which is an ENA requirement for contig-level submissions. Only FASTA files that pass validation are retained for downstream processing and submission. -### RNA presence detection +### tRNA and rRNA genes detection -The workflow only runs internal RNA detection for entries where the `RNA_presence` column is empty. If a value is already supplied in the samplesheet, that value is passed through unchanged. +The workflow only runs tRNA and rRNA genes detection for entries where the `RNA_presence` column is empty. If a value is already supplied in the samplesheet, that value is passed through unchanged. -RNA detection is implemented through the `RNA_DETECTION` subworkflow and combines: +The resulting `RNA_presence` value is a decisive factor affecting MISAG/MIMAG assembly quality classification of a genome. + +tRNA and rRNA genes detection is implemented through the `RNA_DETECTION` subworkflow and combines: - `barrnap` for rRNA prediction - `tRNAscan-SE` for tRNA prediction @@ -42,11 +69,7 @@ RNA detection is implemented through the `RNA_DETECTION` subworkflow and combine `barrnap` is run in bacterial mode (`"bac"`). -The custom parser then scans the GFF output and keeps only the following ribosomal RNA subunits: - -- `16S_rRNA` -- `23S_rRNA` -- `5S_rRNA` +The custom parser then scans the GFF output and keeps only the following rRNA subunits: `16S_rRNA`, `23S_rRNA`, `5S_rRNA`. For each detected feature, the recovered length is calculated as: @@ -66,7 +89,7 @@ $$ If multiple hits are found for the same subunit, the workflow keeps the best recovered percentage for that subunit. -A subunit is considered present when its best recovered percentage is greater than or equal to `--rrna_limit`. The current default is `80`. +A subunit is considered present when its best recovered percentage is greater than or equal to `--rrna_limit`. The current default is `80`. This threshold is arbitrary, as no strict consensus length is defined by ENA or MISAG/MIMAG for considering an rRNA gene present. #### tRNA detection @@ -79,16 +102,16 @@ The parser sums the counts assigned to the 20 standard amino-acid isotypes: - `Leu`, `Lys`, `Met`, `Phe`, `Pro` - `Ser`, `Thr`, `Trp`, `Tyr`, `Val` -The total number of predicted tRNAs is compared against `params.trna_limit`. The current default is `18`. +The total number of predicted tRNAs is compared against `params.trna_limit`. The current default is `18`. This value is defined by the MISAG/MIMAG standard and must not be modified. -#### Final RNA presence decision +#### Final `RNA_presence` decision The final decision stored in `RNA_presence` is: -- `Yes` when at least 18 tRNAs are detected and all three required rRNA subunits pass the recovery threshold -- `No` otherwise +- `True` when at least 18 tRNAs are detected and all three required rRNA subunits pass the recovery threshold +- `False` otherwise -The result is written as a two-column TSV file containing the genome identifier and the final `Yes`/`No` decision, and this value is then merged back into the submission metadata. +The final `True`/`False` decision is then merged back into the submission metadata. ### Genome coverage calculation @@ -98,11 +121,11 @@ Genome coverage values from `coverm genome` output TSV are parsed and merged int ### Taxonomy assignment -If `NCBI_lineage` is already present in the input samlesheet, the value is retained. If it is missing, the workflow runs taxonomy classification using the `CAT_pack` tool. +If `NCBI_lineage` is already present in the input samplesheet, the value is retained. If it is missing, the workflow runs taxonomy classification using the `CAT_pack` tool. Before classification, input FASTA files are normalized to a `.fasta` suffix by `RENAME_FASTA_FOR_CATPACK`. Classification is then run in bin mode (`CAT_pack bins`, followed by `CAT_pack add_names`). -Database input is taken from `--cat_db` when provided; otherwise the workflow uses `--cat_db_download_id` to download and prepare a CATPACK database. The resulting classification table is parsed, and the lineage field is written to `NCBI_lineage`. +Database input is taken from `--cat_db` when provided; otherwise the workflow builds CAT_pack database from NCBI non-redundant protein database (`nr`). See [CAT_pack documentation](https://github.com/MGXlab/CAT_pack#downloading-preconstructed-database-files). There are two options available: `nr` - NCBI non-redundant protein database and `gtdb` (Genome Taxonomy Database (GTDB) proteins). We use NCBI proteins because ENA supports only NCBI format of taxonomy for submission. The resulting classification table is parsed, and the lineage field is written to `NCBI_lineage`. ### Completeness and contamination assessment @@ -111,36 +134,3 @@ Completeness and contamination are evaluated together in a shared genome quality The workflow checks three samplesheet fields: `completeness`, `contamination`, and `stats_generation_software`. If all three are already present, those values are reused. If any of them is missing, the genome is analysed with `CheckM2 predict`. If `--checkm2_db` is supplied and exists, that database is used directly. Otherwise, the workflow downloads a CheckM2 database from Zenodo (using the configured database ID) and then runs prediction. For records that run `CheckM2`, completeness and contamination are extracted from the generated quality report (`quality_report.tsv`) and used `CheckM2` version is recorded as `stats_generation_software`. - -## `ASSEMBLYSUBMIT` methods - -### Overview - -The `ASSEMBLYSUBMIT` workflow: - -1. Reads the samplesheet and associated assembly FASTA files. -2. Validates assembly FASTA files. -3. Reuses `coverage` values supplied in the samplesheet when they are already present. -4. Calculates `coverage` internally for entries where this field is missing and reads are available. -5. Builds the metadata table used for manifest generation. -6. Generates assembly manifests and submits the assemblies to ENA. - -### Assembly FASTA validation - -Assembly FASTA files are validated with `FASTAVALIDATOR` before downstream processing. Each file is checked for FASTA format validity and contig count. A genome must contain at least two contigs to pass validation, which is an ENA requirement for contig-level submissions. - -Only assemblies with successful validation are forwarded to coverage estimation, metadata/manifest generation and submission. - -### Coverage calculation - -If the `coverage` column is already populated in the samplesheet, that value is used directly. - -If `coverage` is missing, the workflow calculates coverage with `coverm contig`. - -`coverm contig` outputs per-contig depth. The workflow then reads this file and calculates the arithmetic mean across all contigs. - -If the per-contig coverage values are $c_1, c_2, \ldots, c_n$, the workflow currently computes assembly coverage as an unweighted mean across contigs: - -$$ -\bar{c} = \frac{1}{n} \sum_{i=1}^{n} c_i -$$ diff --git a/docs/usage.md b/docs/usage.md index 7d3b5ef8..14c38e20 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -4,137 +4,55 @@ > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ +## Table of Contents + +- [Introduction](#introduction) +- [Before you start](#before-you-start) +- [Submission study](#submission-study) +- [Data privacy](#data-privacy) +- [Third party assemblies (TPA)](#third-party-assemblies-tpa) +- [`reads` mode](#reads-mode) +- [`metagenomic_assemblies` mode](#metagenomic_assemblies-mode) +- [`mags` and `bins` modes](#mags-and-bins-modes) +- [Scalability](#scalability) +- [Running the pipeline](#running-the-pipeline) +- [Core Nextflow arguments](#core-nextflow-arguments) +- [Custom configuration](#custom-configuration) +- [Running in the background](#running-in-the-background) +- [Nextflow memory requirements](#nextflow-memory-requirements) + ## Introduction -`nf-core/seqsubmit` is a Nextflow pipeline for submitting metagenomic assemblies, MAGs, bins, and raw reads to ENA. +`nf-core/seqsubmit` is a Nextflow pipeline for submitting sequence data to ENA. + +The pipeline supports the following data types, selected via the `--mode` parameter: -The pipeline supports three workflow paths: +- `reads` — raw sequencing reads +- `metagenomic_assemblies` — metagenomic assemblies +- `mags` — metagenome-assembled genomes (MAGs) +- `bins` — metagenomic bins -- `GENOMESUBMIT` for `--mode mags` and `--mode bins` -- `ASSEMBLYSUBMIT` for `--mode metagenomic_assemblies` -- `READSUBMIT` for `--mode reads` +Each mode has its own samplesheet _structure_, _prerequisites_, and _limitations_ — see the dedicated section for the mode you want to use below. ## Before you start -Before running the pipeline, make sure that: +Before running the pipeline in any mode, make sure that: - Nextflow `>=25.04.0` is available. - You have a Webin account registered at . -- The raw reads used to generate the submitted assemblies have already been submitted to INSDC/ENA and the relevant accessions are available. - -Set your Webin credentials as Nextflow secrets: - -```bash -nextflow secrets set ENA_WEBIN "Webin-XXX" -nextflow secrets set ENA_WEBIN_PASSWORD "XXX" -``` - -## Samplesheet input - -You will need to create a samplesheet with information about the data entries you would like to process before running the pipeline. Use `--input` parameter to specify its location. It has to be a comma-separated file with the structure defined by the execution `--mode`. - -```bash ---input '[path to samplesheet.csv]' -``` - -### `mags` and `bins` modes (`GENOMESUBMIT`) - -Use this samplesheet structure for MAG and bin submission. The input format follows [assets/schema_input_genome.json](../assets/schema_input_genome.json). - -Example: - -```csv title="samplesheet_genomes.csv" -sample,fasta,accession,fastq_1,fastq_2,assembly_software,binning_software,binning_parameters,stats_generation_software,completeness,contamination,genome_coverage,metagenome,co-assembly,broad_environment,local_environment,environmental_medium,RNA_presence,NCBI_lineage -mag_001,data/mag_001.fasta.gz,SRR24458089,,,SPAdes 3.15.5,MetaBAT2 2.15,default,CheckM2 1.0.1,92.81,1.09,66.04,sediment metagenome,No,marine,cable bacteria,marine sediment,No,d__Bacteria;p__Proteobacteria;s__ -``` - -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. - -| Column | Description | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset (used as `meta.id` throughout the pipeline). | -| `fasta` | Path to MAG/bin contigs in FASTA format compressed with `gzip`. All names of the FASTA files must be unique to prevent pipeline errors. | -| `accession` | ENA accession of the run or metagenomic assembly used to generate the MAG/bin. | -| `fastq_1` | Path to the read file in FASTQ format used to generate the source metagenomic assembly. Required if `genome_coverage` is not provided. | -| `fastq_2` | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. | -| `assembly_software` | Tool name and version that were used to generate the source metagenomic assembly. | -| `binning_software` | Binning tool, including version, that was used to generate the bins. | -| `binning_parameters` | Arguments that were used during binning. | -| `stats_generation_software` | Tool, including version, that was used to calculate completeness and contamination. | -| `completeness` | Genome completeness value. | -| `contamination` | Genome contamination value. | -| `genome_coverage` | Estimated average sequencing depth across the genome. If the value is missing, it is computed automatically during pipeline execution when reads are provided. | -| `metagenome` | Registered metagenome taxonomic identifier or name that matches an existing ENA taxonomy entry. For more details see https://ena-docs.readthedocs.io/en/latest/faq/taxonomy.html | -| `co-assembly` | Whether a co-assembly strategy was used for the initial metagenomic assembly generation. Options: Yes or No. | -| `broad_environment` | Broad ecological context of the sample, for example 'marine biome', 'desert biome'. It is recommended to use subclasses of EnvO 'biome' class (http://purl.obolibrary.org/obo/ENVO_00000428) | -| `local_environment` | Local environmental context of the sample, for example 'tropical dry broadleaf forest biome', 'marine abyssal zone biome'. It is recommended to use EnvO terms which are of smaller spatial grain than your entry for "broad-scale environmental context". | -| `environmental_medium` | Material displaced by the sample, or the material in which the sample was embedded before sampling, for example 'mucus', 'lake water'. It is recommended to use subclasses of EnvO 'environmental material' class (http://purl.obolibrary.org/obo/ENVO_00010483). | -| `RNA_presence` | Presence or absence of the 23S, 16S, and 5S rRNA genes and at least 18 tRNAs. This is used for MISAG/MIMAG assembly quality classification. Options: Yes or No. | -| `NCBI_lineage` | NCBI taxonomy lineage of the genome. Can be composted of either numerical IDs or taxon names separated by ";". | -> [!NOTE] -> More information about envioronment tags can be found at checklists [ERC000050](https://www.ebi.ac.uk/ena/browser/view/ERC000050) for bins and [ERC000047](https://www.ebi.ac.uk/ena/browser/view/ERC000047) for MAGs under the field names "broad-scale environmental context", "local environmental context", and "environmental medium". - -### `metagenomic_assemblies` mode (`ASSEMBLYSUBMIT`) - -Use this samplesheet structure for metagenomic assembly submission. The input format follows [assets/schema_input_assembly.json](../assets/schema_input_assembly.json). - -Provide either read files (`fastq_1`, optionally `fastq_2`) or a `coverage` value for each row. If `coverage` is missing and reads are provided, the workflow calculates average coverage automatically. - -Example: - -```csv title="samplesheet_assembly.csv" -sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -assembly_001,data/assembly_001.fasta.gz,data/assembly_001_R1.fastq.gz,data/assembly_001_R2.fastq.gz,,ERR011322,SPAdes,3.15.5 -assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9 -``` - -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. - -| Column | Description | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `sample` | A unique identifier for this data entry. Must be globally unique within the input dataset (used as `meta.id` throughout the pipeline). | -| `fasta` | Path to assembly contigs in FASTA format compressed with `gzip`. | -| `fastq_1` | Path to the read file in FASTQ format used to generate the metagenomic assembly. Required if `coverage` is not provided. | -| `fastq_2` | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. | -| `coverage` | Estimated sequencing depth of the assembly. If this value is missing, it is computed automatically during pipeline execution when reads are provided. | -| `run_accession` | ENA run accession for the reads used to generate the metagenomic assembly. Reads must already be submitted to ENA. | -| `assembler` | Name of the assembler software used to generate the assembly. | -| `assembler_version` | Version of the assembler software used to generate the assembly. | - -An example file is available at [assets/samplesheet_assembly.csv](../assets/samplesheet_assembly.csv). + Set your Webin credentials as Nextflow secrets: -### `reads` mode (`READSUBMIT`) + ```bash + nextflow secrets set ENA_WEBIN "Webin-XXX" + nextflow secrets set ENA_WEBIN_PASSWORD "XXX" + ``` -Use this samplesheet structure for raw sequencing reads submission. The input format follows [assets/schema_input_reads.json](../assets/schema_input_reads.json). + Make sure to replace the values above with your own credentials. -Example: +- Provide either a study accession or a study registration metadata file for the study the submission will be associated with. See the [Submission study](#submission-study) section below for details. -```csv title="samplesheet_reads.csv" -sample,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description -illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ -pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing -``` - -> [!IMPORTANT] -> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown. - -| Column | Type | Required | Description | -| ------------------- | --------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `sample` | str | Yes | Unique identifier of this particular data entry. Used as an experiment name. | -| `sample_accession` | str | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. | -| `fastq_1` | file path | Yes | Path to forward reads in FASTQ format (optionally gzipped). | -| `fastq_2` | file path | No | Path to reverse reads for paired-end data. Leave empty for single-end reads. | -| `platform` | str | Yes | Sequencing platform. Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation) for complete list. | -| `instrument` | str | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". | -| `library_source` | str | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. | -| `library_selection` | str | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation) for complete list. | -| `library_strategy` | str | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation) for complete list. | -| `insert_size` | number | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | -| `library_name` | str | No | Descriptive library name (optional). | -| `description` | str | No | Free-text description of the experiment (optional). | +- Depending on the chosen mode, samples, reads, or metagenomic assemblies must be pre-submitted to ENA to obtain the corresponding accessions, which you then reference in your submission. Refer to the relevant mode section below for details. ## Submission study @@ -148,7 +66,7 @@ If you already have an ENA study, pass its accession (starting with `PRJ` or `ER --submission_study PRJEB12345 ``` -You can create a study manually via the [Webin Portal](https://www.ebi.ac.uk/ena/submit/webin/login) and then use the assigned accession here. +You can create a study manually via the [Webin Portal](https://www.ebi.ac.uk/ena/submit/webin/login) following [these instructions](https://ena-docs.readthedocs.io/en/latest/submit/study/interactive.html) and then use the assigned study accession. ### Option 2 — Register a new study automatically @@ -198,95 +116,310 @@ study-soil-2026 Soil microbiome study Survey of soil microbiota | `existing_study_type` | No | ENA study type (e.g. `Metagenomics`, `Other`). | | `new_study_type` | No | Custom study type. Only used when `existing_study_type` is set to `Other`. | -## Database preparation (`mags` / `bins`) +## Data privacy -The `GENOMESUBMIT` workflow uses `CheckM2` and `CAT_pack` that require specialized databases for completeness/contamination assessment and taxonomy assignment. +You can reference private ENA data if you have access to it via your Webin account (it was submitted previously under your credentials). To use private data accessions in your submission metadata, specify the `--is_private` flag. The pipeline will then use your provided Webin credentials to fetch the required metadata. -You can either provide pre-existing databases or let the pipeline prepare them during execution. +You can also control the privacy of your submitted data: -- `CheckM2`: - - provide the path to local database with `--checkm2_db`, otherwise the pipeline downloads version specified with `--checkm2_db_download_id` (by default `14897628`). +- If you provide an existing study via `--submission_study`, your submission will inherit the same privacy status as that study. +- If no study is provided, the pipeline will register a new one for you. To keep this new study private, you must specify a release date using `--release_date YYYY-MM-DD` (up to 2 years from the current date). -- `CAT_pack`: - - provide the path to local database (containing `tax/` and `db/` folders or `tar.gz` archive) with `--cat_db`, otherwise the pipeline constructs version specified with `--cat_db_download_id` (by default `nr`). +| Example | Source Data | Submission Visibility | Required Arguments | Result | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------- | ---------------------------------------- | ----------------------------------------------------- | +| Assemblies generated from public reads. Assemblies should be public immediately after submission | public | public | – | SUCCESS | +| Assemblies generated from public reads. Assemblies should remain private until 1 July 2028 submission | public | private | `--release_date 2028-07-01` | SUCCESS | +| Bins generated from private reads and assemblies. Bins should be public after submission. User **has access** to the reads and assemblies via Webin account | private | public | `--is_private` | SUCCESS | +| Bins generated from private reads and assemblies. Bins should remain private until 1 July 2028. User **has access** via Webin account | private | private | `--is_private --release_date 2028-07-01` | SUCCESS | +| MAGs generated from private reads and assemblies. MAGs should be public after submission. User **does not have access** to the reads and assemblies via Webin account | private | public | – | ERROR (submission can only reference accessible data) | -See [CAT_pack documentation](https://github.com/MGXlab/CAT_pack) and [CheckM2 documentation](https://github.com/chklovski/CheckM2) for more details on usage and creation of databases. +## Third party assemblies (TPA) -> [!IMPORTANT] -> `CAT_pack` database creation can take significant time. -> -> Reusing an existing database is strongly recommended for repeated runs. -> -> Databases created/downloaded by the pipeline are published under: -> `${params.outdir}/databases/` +A Third PArty (TPA) Assembly record is an assembly or reassembly built from primary sequence data that has already been submitted to an INSDC database (ENA, GenBank, or DDBJ), rather than newly generated sequence data. TPA records must be built entirely from publicly available accessions or public sequencing reads — they cannot incorporate proprietary or unpublished data, and any newly generated sequence they contain must first be released as a primary submission in its own right. -## Running the pipeline +If you use `metagenomic_assemblies`, `mags`, or `bins` mode to submit data assembled from public data you did not generate yourself, use `--upload_tpa` to flag the submission as a TPA. -General command template: +See [ENA's data policies](https://www.ebi.ac.uk/ena/browser/about/policies) for further guidance on when a TPA designation applies to your submission. + +## `reads` mode + +### Prerequisites + +In addition to the general [prerequisites](#before-you-start), for `reads` mode: + +- You need to register the source BioSamples and obtain the sample accessions as described in the [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/samples.html). These accessions must be provided in the `sample_accession` column of the samplesheet. + +### Samplesheet input + +You will need to create a samplesheet with information about the raw reads you would like to submit before running the pipeline. Use `--input` to specify its location. It must be a comma-separated file with the structure that follows [assets/schema_input_reads.json](../assets/schema_input_reads.json). + +Example: + +```csv title="samplesheet_reads.csv" +id,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description +illumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ +pacbio_run_001,SAMEA7654321,data/pacbio_reads.fastq.gz,,PACBIO_SMRT,PacBio Sequel,GENOMIC,RANDOM,WGS,,PacBio_library_002,Long-read sequencing +``` + +| Column | Required | Description | +| ------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | Unique identifier of this particular data entry. Used as an experiment name. | +| `sample_accession` | Yes | ENA sample accession (starting with SAMEA) of the sample used to generate raw reads. Samples must already be submitted to ENA. | +| `fastq_1` | Yes | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). | +| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped). Leave empty for single-end reads. | +| `platform` | Yes | Sequencing platform. Must be one of the ENA controlled vocabulary values listed in [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#metadata-validation). Supported values: `ILLUMINA`, `PACBIO_SMRT`, `OXFORD_NANOPORE`, `ION_TORRENT`, `CAPILLARY`, `DNBSEQ`, `ELEMENT`, `GENAPSYS`, `GENEMIND`, `HELICOS`, `LS454`, `BGISEQ`, `ULTIMA`, `VELA_DIAGNOSTICS`. | +| `instrument` | Yes | Sequencer model, e.g. "Illumina HiSeq 2000", "PacBio Sequel", "MinION". See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-instrument). | +| `library_source` | Yes | Library source type. Options: `GENOMIC`, `METAGENOMIC`, `TRANSCRIPTOMIC`, `METAGENOMIC SINGLE CELL`, `TRANSCRIPTOMIC SINGLE CELL`, `SYNTHETIC`, `VIRAL RNA`, `OTHER`. ENA [documantation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-source). | +| `library_selection` | Yes | Library selection method. Options: `RANDOM`, `PCR`, `RANDOM PCR`, `RT-PCR`, `MF`, `cDNA`, `cDNA_randomPriming`, `cDNA_oligo_dT`, `PolyA`, `Inverse rRNA`, `ChIP`, `MNase`, `DNase`, `Hybrid Selection`, etc. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-selection) for complete list. | +| `library_strategy` | Yes | Library strategy. Options: `WGS`, `WGA`, `WXS`, `RNA-Seq`, `miRNA-Seq`, `ncRNA-Seq`, `EST`, `Hi-C`, `ATAC-seq`, `WCS`, `RAD-Seq`, `CLONE`, `AMPLICON`, `POOLCLONE`, `etc`. See [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads/webin-cli.html#permitted-values-for-library-strategy) for complete list. | +| `insert_size` | No | Fragment/insert size for paired-end reads (e.g., 500 for 500 bp inserts). Leave empty if not applicable. | +| `library_name` | No | Descriptive library name (optional). | +| `description` | No | Free-text description of the experiment (optional). | + +### `reads` parameters and example + +`reads` mode does not add any parameters beyond the [common parameters](#running-the-pipeline). + +Example `reads` mode upload to TEST server, run command with docker: ```bash nextflow run nf-core/seqsubmit \ - -profile \ - --mode \ - --input \ - --centre_name \ + -profile docker \ + --mode reads \ + --input samplesheet_reads.csv \ --submission_study \ - --outdir + --webincli_mode submit \ + --test_upload \ + --outdir reads_submission/ +``` + +### Limitations + +`reads` mode does not support the following: + +- **Reads in non-FASTQ formats**: Only reads provided as FASTQ (`.fastq.gz`) can be submitted. Other formats (e.g. BAM) are not supported as input. + +## `metagenomic_assemblies` mode + +### Prerequisites + +In addition to the general [prerequisites](#before-you-start), for `metagenomic_assemblies` mode: + +- The raw reads used to generate the assemblies have to be submitted to INSDC/ENA to obtain the corresponding run accessions. Refer to the [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads.html) for details. You can use `seqsubmit` in `reads` mode to do this. Assigned run accessions must be provided in the `run_accession` column of the samplesheet. + +### Samplesheet input + +You will need to create a samplesheet with information about the metagenomic assemblies you would like to submit before running the pipeline. Use `--input` to specify its location. It must be a comma-separated file with the structure that follows [assets/schema_input_assembly.json](../assets/schema_input_assembly.json). + +Example: + +```csv title="samplesheet_assembly.csv" +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +assembly_001,data/assembly_001.fasta.gz,data/assembly_001_R1.fastq.gz,data/assembly_001_R2.fastq.gz,,ERR011322,SPAdes,3.15.5 +assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9 ``` -Key parameters: +| Column | Required | Description | +| ------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | A unique identifier for this data entry. Must be globally unique within the input dataset. | +| `fasta` | Yes | Path to assembly contigs in FASTA format compressed with `gzip`. | +| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required if `coverage` is not provided. | +| `fastq_2` | No | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. | +| `coverage` | Conditional | Estimated sequencing depth of the assembly. If this value is missing, it is computed automatically during pipeline execution when reads are provided. | +| `run_accession` | Yes | ENA run accession for the reads used to generate the metagenomic assembly. Reads must already be submitted to ENA. | +| `assembler` | Yes | Name of the assembler software used to generate the assembly, e.g. MEGAHIT, SPAdes, metaSPAdes, Flye, etc. | +| `assembler_version` | Yes | Version of the assembler software used to generate the assembly, e.g. 1.0.0. | -| Parameter | Description | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `--mode` | Submission type. Supported values are `mags`, `bins`, `metagenomic_assemblies`, and `reads`. | -| `--input` | Path to the samplesheet describing the data to submit. | -| `--submission_study` | ENA study accession (PRJ/ERP) to submit the data to. For metagenomic assemblies, this is the paper's ENA Assembly Project accession. | -| `--centre_name` | Name of the submitter's organisation. | -| `--test_upload` | Submit to the ENA TEST server instead of the LIVE server. | -| `--webincli_mode` | Webin-CLI mode for ENA interaction: `submit` uploads data, `validate` performs validation only. | -| `--upload_tpa` | Mark assemblies as third party assemblies when required. | +Provide either read files (`fastq_1`, optionally `fastq_2`) or a `coverage` value for each row. If `coverage` is missing and reads are provided, the workflow calculates average coverage with CoverM. See the [Methods documentation](docs/methods.md) for more information on how coverage is calculated. -Test example for `mags` run with docker: +### `metagenomic_assemblies` parameters and example + +In addition to the [common parameters](#running-the-pipeline): + +| Parameter | Required | Description | +| -------------- | -------- | --------------------------------------------------------------------------------------------------------------- | +| `--upload_tpa` | No | Mark assemblies as [third party assemblies (TPA)](#third-party-assemblies-tpa) when required. Default: `false`. | + +Example `metagenomic_assemblies` mode upload to TEST server, run command with docker: ```bash nextflow run nf-core/seqsubmit \ -profile docker \ - --mode mags \ - --input assets/samplesheet_genomes.csv \ + --mode metagenomic_assemblies \ + --input samplesheet_assembly.csv \ --submission_study \ - --centre_name TEST_CENTER \ --webincli_mode submit \ --test_upload \ - --outdir results/validate_mags + --outdir assembly_submission ``` -Test example for `metagenomic_assemblies` run with docker: +Example `metagenomic_assemblies` mode upload to LIVE server, run command with docker: ```bash nextflow run nf-core/seqsubmit \ -profile docker \ --mode metagenomic_assemblies \ - --input assets/samplesheet_assembly.csv \ + --input samplesheet_assembly.csv \ --submission_study \ - --centre_name TEST_CENTER \ + --test_upload false \ --webincli_mode submit \ - --test_upload \ - --outdir results/validate_assemblies + --outdir assemblies_submission/ +``` + +### Limitations + +`metagenomic_assemblies` mode does not yet support the following: + +- **Co-assemblies**: Metagenomic co-assemblies are not supported. The pipeline currently expects each assembly to be derived from a single run. Tracked in [issue #61](https://github.com/nf-core/seqsubmit/issues/61), addressed in [PR #66](https://github.com/nf-core/seqsubmit/pull/66). + +- **Submission of an assembly without submitting the raw reads**: Assemblies that have no associated reads accession (`run_accession` column in the samplesheet) are not supported, since the pipeline relies on that lineage to build the required metadata. Tracked in [issue #29](https://github.com/nf-core/seqsubmit/issues/29). + +- **Single-contig assemblies**: Assemblies that consist of only one contig cannot be uploaded through this pipeline. ENA classifies these as "chromosomal assemblies", which follow a different submission procedure with different metadata requirements. If you have a single-contig assembly to submit, please contact [ENA support](https://www.ebi.ac.uk/ena/browser/support) for guidance. + +- **Coverage calculation from long reads**: `coverage` calculation via CoverM has only been tested and validated with short reads (e.g. Illumina). It has not been tested with long reads (PacBio, Nanopore). Exercise caution when processing data generated from long reads, as issues may occur. + +## `mags` and `bins` modes + +### Prerequisites + +In addition to the general [prerequisites](#before-you-start), for `mags` or `bins` mode: + +- Either the raw reads alone, or the raw reads together with the metagenomic assemblies, used to generate the MAGs/bins must be submitted to INSDC/ENA to obtain the corresponding run or assembly accessions. Refer to the [ENA documentation](https://ena-docs.readthedocs.io/en/latest/submit/reads.html) for details. You can use the pipeline in `reads` and `metagenomic_assemblies` mode to do this. Assigned run or assembly accessions must be provided in the `accession` column of the samplesheet. + +### Samplesheet input + +You will need to create a samplesheet with information about the MAGs/bins you would like to submit before running the pipeline. Use `--input` to specify its location. It must be a comma-separated file with the structure that follows [assets/schema_input_genome.json](../assets/schema_input_genome.json). + +Example: + +```csv title="samplesheet_genomes.csv" +id,fasta,accession,fastq_1,fastq_2,assembly_software,binning_software,binning_parameters,stats_generation_software,completeness,contamination,genome_coverage,metagenome,co-assembly,broad_environment,local_environment,environmental_medium,RNA_presence,NCBI_lineage +mag_001,data/mag_001.fasta.gz,SRR24458089,,,SPAdes 3.15.5,MetaBAT2 2.15,default,CheckM2 1.0.1,92.81,1.09,66.04,sediment metagenome,false,marine,cable bacteria,marine sediment,false,d__Bacteria;p__Proteobacteria;s__ ``` -Test example for `reads` run with docker: +| Column | Required | Description | +| --------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | A unique identifier for this data entry. Must be globally unique within the input dataset. | +| `fasta` | Yes | Path to MAG/bin sequences in FASTA format compressed with `gzip`. All names of the FASTA files must be unique to prevent pipeline errors. File should contain at least one sequence. | +| `accession` | Yes | ENA accession of the run or metagenomic assembly used to generate the MAG/bin. Reads or assembly must already be submitted to ENA. | +| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped) used to generate the source metagenomic assembly. Required if `genome_coverage` is not provided. | +| `fastq_2` | No | Path to reverse reads in FASTQ format for paired-end data (optionally gzipped) used to generate the source metagenomic assembly. Leave empty for single-end reads. | +| `assembly_software` | Yes | Tool name and version that were used to generate the source metagenomic assembly, e.g. MEGAHIT_v1.0, SPAdes_v4.0.0, metaSPAdes_v3.15.0. | +| `binning_software` | Yes | Tool name and version that was used to bin data, e.g. MetaBAT2_v1.0, VAMB_2.0, MaxBin2_v1, CONCOCT_v3.0, SemiBin2_v1, COMEBin_v2.3.4, etc. | +| `binning_parameters` | Yes | Arguments used to bin data different from default, e.g. -min_contig_length 1000, otherwise use 'default'. | +| `stats_generation_software` | No | Tool, including version, that was used to calculate completeness and contamination, e.g. CheckM2_v1.0.0. | +| `completeness` | No | MAG/bin completeness score: the 0<= ratio <= 100 of observed single-copy marker genes to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined. | +| `contamination` | No | MAG/bin contamination score: the 0<= ratio <= 100 of observed single-copy marker genes in ≥2 copies to total single-copy marker genes in chosen marker gene set (%). ENA docs: https://ena-docs.readthedocs.io/en/latest/faq/metagenomes.html#how-is-the-quality-of-a-metagenomic-assembly-defined. | +| `genome_coverage` | Conditional | Estimated average sequencing depth (>0) across the genome. If the value is missing, it is computed automatically during pipeline execution when reads are provided. | +| `metagenome` | Yes | Registered metagenome taxonomic identifier or name that matches an existing ENA taxonomy entry. It needs to be listed in the taxonomy tree https://www.ebi.ac.uk/ena/browser/view/408169?show=tax-tree (you might need to press "Tax tree - Show" in the right most section of the page). Full list can also be found https://github.com/EBI-Metagenomics/genome_uploader/blob/main/genomeuploader/constants.py#L22. | +| `co-assembly` | Yes | Whether a co-assembly strategy was used for the initial metagenomic assembly generation. Options: true or false. | +| `broad_environment` | Yes | Broad ecological context of the sample, for example 'marine biome', 'desert biome'. It is recommended to use subclasses of EnvO 'biome' class (http://purl.obolibrary.org/obo/ENVO_00000428). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS. | +| `local_environment` | Yes | Local environmental context of the sample, for example 'tropical dry broadleaf forest biome', 'marine abyssal zone biome'. It is recommended to use EnvO terms which are of smaller spatial grain than your entry for "broad-scale environmental context". Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS. | +| `environmental_medium` | Yes | Material surrounding and in contact with the sample, for example 'mucus', 'lake water'. It is recommended to use subclasses of EnvO 'environmental material' class (http://purl.obolibrary.org/obo/ENVO_00010483). Documentation: https://github.com/EnvironmentOntology/envo/wiki/Using-ENVO-with-MIxS. | +| `RNA_presence` | No | Presence or absence of the 23S, 16S, and 5S rRNA genes and at least 18 tRNAs. This is used for MISAG/MIMAG assembly quality classification. Options: true or false. | +| `NCBI_lineage` | No | Full NCBI lineage - format: x;y;z. For example, the lineage for E. coli can be: 'Bacteria;Pseudomonadati;Pseudomonadota;Gammaproteobacteria;Enterobacterales;Enterobacteriaceae;Escherichia' or '2;1224;1236;91347;543;561;562'. For more info check https://www.ncbi.nlm.nih.gov/datasets/docs/v2/data-processing/taxonomy-processing/taxonomy/. | + +> [!NOTE] +> More information about environment tags can be found at checklists [ERC000050](https://www.ebi.ac.uk/ena/browser/view/ERC000050) for bins and [ERC000047](https://www.ebi.ac.uk/ena/browser/view/ERC000047) for MAGs under the field names "broad-scale environmental context", "local environmental context", and "environmental medium". + +If `genome_coverage`, `stats_generation_software`, `completeness`, `contamination`, `RNA_presence`, or `NCBI_lineage` are missing, the workflow can calculate or infer them when the required inputs are available. See the [Methods documentation](docs/methods.md) for more information on how metadata statistics are obtained. + +### Preparing the CheckM2 and CAT_pack databases + +`mags` and `bins` submission modes use CheckM2 and CAT_pack tools that require specialized databases for completeness/contamination assessment and taxonomy assignment. + +You can either provide pre-existing databases or let the pipeline prepare them during execution. + +As database preparation can take significant time, we strongly recommend downloading them locally and storing them in a local cache folder for reuse across runs. +In particular, the CAT_pack database is extremely large and may take a long time to download and compute. + +- CheckM2: + - provide the path to a local database with `--checkm2_db`, otherwise the pipeline downloads the version specified with `--checkm2_db_download_id` (by default Zenodo accession `14897628`). + +- CAT_pack: + - provide the path to a local database (containing `tax/` and `db/` folders or a `tar.gz` archive) with `--cat_db`, otherwise the pipeline builds CAT_pack DB from NCBI non-redundant protein database (`nr`). + +See the [CAT_pack documentation](https://github.com/MGXlab/CAT_pack) and [CheckM2 documentation](https://github.com/chklovski/CheckM2) for more details on usage and creation of databases. + +Databases created/downloaded by the pipeline are published under: +`/databases/`. In subsequent pipeline runs you can reuse them using `--cat_db /databases/cat_pack/` and `--checkm2_db /databases/checkm2/`. + +### `mags` and `bins` parameters and example + +In addition to the [common parameters](#running-the-pipeline): + +| Parameter | Required | Description | +| -------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--upload_tpa` | No | Mark assemblies as [third party assemblies (TPA)](#third-party-assemblies-tpa) when required. Default: `false`. | +| `--checkm2_db` | No | Path to a pre-downloaded CheckM2 database. If omitted, downloads the version set by `--checkm2_db_download_id`. Default: `14897628` (Zenodo accession). | +| `--cat_db` | No | Path to a local CAT_pack database. If omitted, builds CAT_pack DB from NCBI non-redundant protein database (`nr`). | + +Example `mags` mode upload to TEST server, run command with docker: ```bash nextflow run nf-core/seqsubmit \ -profile docker \ - --mode reads \ - --input samplesheet_reads.csv \ + --mode mags \ + --input samplesheet_genomes.csv \ --submission_study \ + --centre_name \ --webincli_mode submit \ --test_upload \ - --outdir results/validate_reads + --outdir mags_submission/ ``` +### Limitations + +`mags` and `bins` modes do not yet support the following: + +- **Co-assemblies**: MAGs/bins generated from co-assemblies are not supported. The pipeline currently expects each MAG/bin to be derived from an assembly generated from a single run. Tracked in [issue #61](https://github.com/nf-core/seqsubmit/issues/61). + +- **Eukaryotic and viral bins/MAGs**: Submitting eukaryotic or viral genomes that require additional metadata generation is not supported. For such genomes tRNA and rRNA prediction, quality assessment (completeness/contamination estimation) and taxonomy assignment are not implemented. If all the required metadata (`completeness`, `contamination`, `stats_generation_software`, `RNA_presence` and `NCBI_lineage`) is already provided in the samplesheet, eukaryotic and viral bins/MAGs can still be submitted normally. Tracked in [issue #40](https://github.com/nf-core/seqsubmit/issues/40) (eukaryotic) and [issue #63](https://github.com/nf-core/seqsubmit/issues/63) (viral). + +- **Submission of a MAG/bin without submitting the assembly or raw reads**: If a MAG/bin cannot be traced back to an assembly accession or a reads accession — i.e. only a sample accession is available — it cannot currently be submitted, since the pipeline relies on that lineage to build the required metadata. Tracked in [issue #87](https://github.com/nf-core/seqsubmit/issues/87). + +- **Single-contig MAGs/bins**: MAGs/bins that consist of only one contig cannot be uploaded through this pipeline. ENA classifies these as "chromosomal assemblies", which follow a different submission procedure with different metadata requirements. If you have a single-contig MAG/bin to submit, please contact [ENA support](https://www.ebi.ac.uk/ena/browser/support) for guidance. + +- **Coverage calculation from long reads**: `genome_coverage` calculation via CoverM has only been tested and validated with short reads (e.g. Illumina). It has not been tested with long reads (PacBio, Nanopore). Exercise caution when processing data generated from long reads, as issues may occur. + +## Scalability + +Some processes in this pipeline are resource intensive — in particular coverage calculation (CoverM), quality assessment (CheckM2), and taxonomic classification of MAGs/bins (BAT). For real-world data, these steps should be run on an HPC cluster or another compute environment with sufficient CPU, memory, and job scheduling capacity, rather than on a single local machine. + +Depending on the resources available on your machine or cluster, you will likely need to adapt the Nextflow configuration to limit the maximum number of concurrently running jobs and other resource settings using [resourceLimits](https://nf-co.re/docs/developing/institutional-profiles/configuration#resource-limits) in a custom config, so that the pipeline does not try to claim more resources than are actually available. See the [nf-core configuration documentation](https://nf-co.re/docs/usage/getting_started/configuration) for guidance on writing a custom config for your infrastructure. + +When submitting a large number of records, be aware that ENA recommends limiting submissions to 5000 per day to avoid overfilling their processing queue. If you need to submit more than this, consider splitting your samplesheet and spreading the submission across several days. + +## Running the pipeline + +General command template: + +```bash +nextflow run nf-core/seqsubmit \ + -profile \ + --mode \ + --input \ + --centre_name \ + --submission_study \ + --outdir +``` + +Parameters common to all modes: + +| Parameter | Required | Description | +| ------------------------------------------ | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--mode` | Yes | Type of the data to be submitted. Options: `mags`, `bins`, `metagenomic_assemblies`, `reads`. | +| `--input` | Yes | Path to samplesheet describing the data to be submitted (supported formats: csv, tsv, yaml, yml, json). Columns/fields depend on the pipeline mode. See [usage docs](https://nf-co.re/seqsubmit/usage#samplesheet-input). | +| `--outdir` | Yes | Path to the output directory for pipeline results. You have to use absolute paths to storage on Cloud infrastructure. | +| `--submission_study` OR `--study_metadata` | Yes (one of) | ENA study accession (PRJ/ERP) to submit the data to OR metadata file in JSON/TSV/CSV format to register a new study. | +| `--centre_name` | Yes | Name of the submitter's organisation (mandatory for broker accounts). | +| `--test_upload` | No | Submit to the ENA TEST server instead of the LIVE server. Use that flag for development purposes or for validation of your submissions. Submission would be done to TEST ENA server instead of LIVE server. TEST server holds data 24h and removes afterwords. LIVE server does not have option to modify your submission, you can only suppress data and re-upload. Default: `true`. | +| `--webincli_mode` | No | Webin-CLI mode for ENA interaction: `submit` uploads data, `validate` performs validation only. `validate` mode checks correctness of manifest files, data access and validity. `submit` mode does validation and push data to ENA server. Default: `submit`. | +| `--is_private` | No | Use if you are referencing private data accessions in your submission. Set to true when the referenced project/reads/assembly are still private in ENA, so metadata is fetched via authenticated access. Does NOT affect the privacy of your own submission, use --release_date for that. Default: `false`. | +| `--release_date` | No | Use if you want to keep your data private in ENA after submission until a particular date (format: YYYY-MM-DD). Data stays private until this date, then is automatically released and indexed by ENA. Must be no more than 2 years from today. Omit to release the study publicly immediately. | + +Each mode also has additional parameters and example commands — see its own "`` parameters and example" subsection above. + If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. diff --git a/main.nf b/main.nf index f741919f..1c9223ab 100644 --- a/main.nf +++ b/main.nf @@ -55,7 +55,6 @@ workflow NFCORE_SEQSUBMIT { params.checkm2_db, params.checkm2_db_download_id, params.cat_db, - params.cat_db_download_id, params.centre_name, params.upload_tpa, params.test_upload, @@ -123,7 +122,9 @@ workflow { params.mode, params.help, params.help_full, - params.show_hidden + params.show_hidden, + params.submission_study, + params.study_metadata ) // diff --git a/modules/local/count_rna/main.nf b/modules/local/count_rna/main.nf index b2594b7c..2bf5758a 100644 --- a/modules/local/count_rna/main.nf +++ b/modules/local/count_rna/main.nf @@ -20,15 +20,19 @@ process COUNT_RNA { tuple val(meta), path("*rna_decision.tsv"), emit: rna_decision tuple val("${task.process}"), val('python'), eval('python --version 2>&1 | sed "s/Python //g"'), topic: versions + when: + task.ext.when == null || task.ext.when + script: + def prefix = task.ext.prefix ?: "${meta.id}" """ count_rna.py \\ --trna ${trnas_stats} \\ --rrna ${rrna_gff} \\ - --name ${meta.id} \\ + --name ${prefix} \\ --trna-limit ${min_trna_count} \\ --rrna-limit ${min_rrna_percentage} \\ - --output ${meta.id}_rna_decision.tsv + --output ${prefix}_rna_decision.tsv """ stub: diff --git a/modules/local/count_rna/meta.yml b/modules/local/count_rna/meta.yml index d7e92f97..4b505cfe 100644 --- a/modules/local/count_rna/meta.yml +++ b/modules/local/count_rna/meta.yml @@ -49,10 +49,6 @@ output: Tab-separated file containing genome ID and decision (Yes/No) on whether sufficient RNA genes are present pattern: "*_rna_decision.tsv" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@KateSakharova" diff --git a/modules/local/count_rna/tests/main.nf.test b/modules/local/count_rna/tests/main.nf.test index d6218399..657d437a 100644 --- a/modules/local/count_rna/tests/main.nf.test +++ b/modules/local/count_rna/tests/main.nf.test @@ -31,7 +31,7 @@ nextflow_process { { assert process.out.rna_decision.size() == 1 }, { def rnaDecisionContent = path(process.out.rna_decision[0][1]).text - assert rnaDecisionContent.contains("Yes") + assert rnaDecisionContent.contains("True") } ) } diff --git a/modules/local/count_rna/tests/main.nf.test.snap b/modules/local/count_rna/tests/main.nf.test.snap index 46ab55ab..047809f3 100644 --- a/modules/local/count_rna/tests/main.nf.test.snap +++ b/modules/local/count_rna/tests/main.nf.test.snap @@ -6,11 +6,11 @@ { "id": "test" }, - "test_rna_decision.tsv:md5,8162813732f3a353edb6fc3e37e950e1" + "test_rna_decision.tsv:md5,b760b5d61d0c140d55e97a2fd6c25471" ] ] ], - "timestamp": "2026-06-25T21:38:21.745168", + "timestamp": "2026-07-22T17:43:41.374272", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.0" diff --git a/modules/local/create_assembly_metadata_csv/meta.yml b/modules/local/create_assembly_metadata_csv/meta.yml index b1301c72..ffe7be41 100644 --- a/modules/local/create_assembly_metadata_csv/meta.yml +++ b/modules/local/create_assembly_metadata_csv/meta.yml @@ -36,11 +36,6 @@ output: type: file description: CSV file containing assembly metadata to be used as input for the assembly_uploader pattern: "*_assembly_metadata.csv" - - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@ochkalova" diff --git a/modules/local/create_genome_metadata_tsv/main.nf b/modules/local/create_genome_metadata_tsv/main.nf index e78a7afd..d3313f90 100644 --- a/modules/local/create_genome_metadata_tsv/main.nf +++ b/modules/local/create_genome_metadata_tsv/main.nf @@ -18,8 +18,8 @@ process CREATE_GENOME_METADATA_TSV { task.ext.when == null || task.ext.when script: - def co_assembly_value = meta.co_assembly == 'Yes' ? 'True' : 'False' - def rna_presence_value = meta.RNA_presence == 'Yes' ? 'True' : 'False' + def co_assembly_value = meta.co_assembly ? 'True' : 'False' + def rna_presence_value = meta.RNA_presence ? 'True' : 'False' def header = [ 'genome_name', 'genome_path', diff --git a/modules/local/create_genome_metadata_tsv/meta.yml b/modules/local/create_genome_metadata_tsv/meta.yml index 74752ad9..07e08d36 100644 --- a/modules/local/create_genome_metadata_tsv/meta.yml +++ b/modules/local/create_genome_metadata_tsv/meta.yml @@ -35,7 +35,7 @@ input: - broad_environment: Broad environmental context - local_environment: Local environmental context - environmental_medium: Environmental medium description - - RNA_presence: RNA presence status ("Yes" or other) + - RNA_presence: status of tRNA and rRNA genes presence ("Yes" or other) - NCBI_lineage: NCBI taxonomic lineage - fasta: type: file @@ -48,11 +48,6 @@ output: type: file description: Tab-separated TSV file with genome metadata pattern: "*.tsv" - - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@ochkalova" diff --git a/modules/local/create_genome_metadata_tsv/tests/main.nf.test b/modules/local/create_genome_metadata_tsv/tests/main.nf.test index eab3015b..72516f59 100644 --- a/modules/local/create_genome_metadata_tsv/tests/main.nf.test +++ b/modules/local/create_genome_metadata_tsv/tests/main.nf.test @@ -24,11 +24,50 @@ nextflow_process { contamination: '1.2', genome_coverage: '42.7', metagenome: 'marine metagenome', - co_assembly: 'No', + co_assembly: false, broad_environment: 'marine biome', local_environment: 'coastal water', environmental_medium: 'seawater', - RNA_presence: 'Yes', + RNA_presence: true, + NCBI_lineage: 'Bacteria;Proteobacteria;Gammaproteobacteria' + ], + file('https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz') + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("test_create_genome_metadata_tsv_bool_reversed") { + + when { + process { + """ + input[0] = [ + [ + id: 'test_genome', + accession: 'SRR123456', + assembly_software: 'SPAdes_v3.15.5', + binning_software: 'MetaBAT2_v2.15', + binning_parameters: '--minContig 1500', + stats_generation_software: 'CheckM2_v1.0.0', + completeness: '98.4', + contamination: '1.2', + genome_coverage: '42.7', + metagenome: 'marine metagenome', + co_assembly: true, + broad_environment: 'marine biome', + local_environment: 'coastal water', + environmental_medium: 'seawater', + RNA_presence: false, NCBI_lineage: 'Bacteria;Proteobacteria;Gammaproteobacteria' ], file('https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz') @@ -65,11 +104,11 @@ nextflow_process { contamination: '1.2', genome_coverage: '42.7', metagenome: 'marine metagenome', - co_assembly: 'No', + co_assembly: true, broad_environment: 'marine biome', local_environment: 'coastal water', environmental_medium: 'seawater', - RNA_presence: 'Yes', + RNA_presence: false, NCBI_lineage: 'Bacteria;Proteobacteria;Gammaproteobacteria' ], file('https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz') diff --git a/modules/local/create_genome_metadata_tsv/tests/main.nf.test.snap b/modules/local/create_genome_metadata_tsv/tests/main.nf.test.snap index a0730539..1de783c7 100644 --- a/modules/local/create_genome_metadata_tsv/tests/main.nf.test.snap +++ b/modules/local/create_genome_metadata_tsv/tests/main.nf.test.snap @@ -1,4 +1,68 @@ { + "test_create_genome_metadata_tsv_bool_reversed": { + "content": [ + { + "0": [ + [ + { + "id": "test_genome", + "accession": "SRR123456", + "assembly_software": "SPAdes_v3.15.5", + "binning_software": "MetaBAT2_v2.15", + "binning_parameters": "--minContig 1500", + "stats_generation_software": "CheckM2_v1.0.0", + "completeness": "98.4", + "contamination": "1.2", + "genome_coverage": "42.7", + "metagenome": "marine metagenome", + "co_assembly": true, + "broad_environment": "marine biome", + "local_environment": "coastal water", + "environmental_medium": "seawater", + "RNA_presence": false, + "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" + }, + "test_genome_genome_metadata.tsv:md5,0f955fb3bbf7da09ce9f449884693806" + ] + ], + "1": [ + [ + "CREATE_GENOME_METADATA_TSV", + "bash", + "5.2.37(1)-release" + ] + ], + "tsv": [ + [ + { + "id": "test_genome", + "accession": "SRR123456", + "assembly_software": "SPAdes_v3.15.5", + "binning_software": "MetaBAT2_v2.15", + "binning_parameters": "--minContig 1500", + "stats_generation_software": "CheckM2_v1.0.0", + "completeness": "98.4", + "contamination": "1.2", + "genome_coverage": "42.7", + "metagenome": "marine metagenome", + "co_assembly": true, + "broad_environment": "marine biome", + "local_environment": "coastal water", + "environmental_medium": "seawater", + "RNA_presence": false, + "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" + }, + "test_genome_genome_metadata.tsv:md5,0f955fb3bbf7da09ce9f449884693806" + ] + ] + } + ], + "timestamp": "2026-07-23T14:00:57.58563", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.0" + } + }, "test_create_genome_metadata_tsv - stub": { "content": [ { @@ -15,11 +79,11 @@ "contamination": "1.2", "genome_coverage": "42.7", "metagenome": "marine metagenome", - "co_assembly": "No", + "co_assembly": true, "broad_environment": "marine biome", "local_environment": "coastal water", "environmental_medium": "seawater", - "RNA_presence": "Yes", + "RNA_presence": false, "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" }, "test_genome_genome_metadata.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" @@ -45,11 +109,11 @@ "contamination": "1.2", "genome_coverage": "42.7", "metagenome": "marine metagenome", - "co_assembly": "No", + "co_assembly": true, "broad_environment": "marine biome", "local_environment": "coastal water", "environmental_medium": "seawater", - "RNA_presence": "Yes", + "RNA_presence": false, "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" }, "test_genome_genome_metadata.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" @@ -57,7 +121,7 @@ ] } ], - "timestamp": "2026-07-07T13:49:45.599115", + "timestamp": "2026-07-22T23:11:26.015229", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.0" @@ -79,11 +143,11 @@ "contamination": "1.2", "genome_coverage": "42.7", "metagenome": "marine metagenome", - "co_assembly": "No", + "co_assembly": false, "broad_environment": "marine biome", "local_environment": "coastal water", "environmental_medium": "seawater", - "RNA_presence": "Yes", + "RNA_presence": true, "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" }, "test_genome_genome_metadata.tsv:md5,64562a85d19fe5893cad0e89e894058a" @@ -109,11 +173,11 @@ "contamination": "1.2", "genome_coverage": "42.7", "metagenome": "marine metagenome", - "co_assembly": "No", + "co_assembly": false, "broad_environment": "marine biome", "local_environment": "coastal water", "environmental_medium": "seawater", - "RNA_presence": "Yes", + "RNA_presence": true, "NCBI_lineage": "Bacteria;Proteobacteria;Gammaproteobacteria" }, "test_genome_genome_metadata.tsv:md5,64562a85d19fe5893cad0e89e894058a" @@ -121,7 +185,7 @@ ] } ], - "timestamp": "2026-07-07T13:49:37.674712", + "timestamp": "2026-07-22T23:11:19.939886", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.0" diff --git a/modules/local/ena_webin_cli_wrapper/main.nf b/modules/local/ena_webin_cli_wrapper/main.nf index 6d62fda9..7a689619 100644 --- a/modules/local/ena_webin_cli_wrapper/main.nf +++ b/modules/local/ena_webin_cli_wrapper/main.nf @@ -26,6 +26,9 @@ process ENA_WEBIN_CLI_WRAPPER { tuple val("${task.process}"), val('ena-webin-cli'), eval('ena-webin-cli -version'), topic: versions tuple val("${task.process}"), val('mgnify-pipelines-toolkit'), eval('python -c "import importlib.metadata; print(importlib.metadata.version(\'mgnify-pipelines-toolkit\'))"'), topic: versions + when: + task.ext.when == null || task.ext.when + script: def args = task.ext.args ?: "" def prefix = task.ext.prefix ?: "${meta.id}" @@ -40,4 +43,13 @@ process ENA_WEBIN_CLI_WRAPPER { ${test_flag} \\ ${args} """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + if (webincli_mode == "submit") { + """ + touch ${prefix}_accessions.tsv + """ + } + // there is no file for mode=validate } diff --git a/modules/local/ena_webin_cli_wrapper/meta.yml b/modules/local/ena_webin_cli_wrapper/meta.yml index e31a1105..62deb914 100644 --- a/modules/local/ena_webin_cli_wrapper/meta.yml +++ b/modules/local/ena_webin_cli_wrapper/meta.yml @@ -47,11 +47,6 @@ output: TSV file containing the accession assigned by ENA for the submitted item. File has two columns: "alias" and "accession". pattern: "*_accessions.tsv" - - versions: - - "versions.yml": - type: file - description: File containing software versions. - pattern: "versions.yml" authors: - "@KateSakharova" diff --git a/modules/local/generate_assembly_manifest/main.nf b/modules/local/generate_assembly_manifest/main.nf index 4cc0121c..90553c07 100644 --- a/modules/local/generate_assembly_manifest/main.nf +++ b/modules/local/generate_assembly_manifest/main.nf @@ -16,7 +16,7 @@ process GENERATE_ASSEMBLY_MANIFEST { output: tuple val(meta), path("${assembly_study}_upload/*.manifest") , emit: manifest - path "versions.yml", emit: versions + tuple val("${task.process}"), val('assembly_uploader'), eval("assembly_manifest --version | sed 's/assembly_uploader //'"), topic: versions when: task.ext.when == null || task.ext.when @@ -37,11 +37,6 @@ process GENERATE_ASSEMBLY_MANIFEST { ${test_flag} \\ ${is_private_flag} \\ ${args} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - assembly_uploader: \$(assembly_manifest --version | sed 's/assembly_uploader //') - END_VERSIONS """ stub: @@ -50,10 +45,5 @@ process GENERATE_ASSEMBLY_MANIFEST { """ mkdir ${assembly_study}_upload/ touch ${assembly_study}_upload/test.manifest - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - assembly_uploader: \$(assembly_manifest --version | sed 's/assembly_uploader //') - END_VERSIONS """ } diff --git a/modules/local/generate_assembly_manifest/meta.yml b/modules/local/generate_assembly_manifest/meta.yml index 6bd102be..1f9e1de5 100644 --- a/modules/local/generate_assembly_manifest/meta.yml +++ b/modules/local/generate_assembly_manifest/meta.yml @@ -50,10 +50,6 @@ output: - manifest: type: file description: Manifest file required for submission with webin-cli. - - versions: - type: file - description: File containing software versions. - pattern: "versions.yml" authors: - "@fxquah" diff --git a/modules/local/generate_assembly_manifest/tests/main.nf.test b/modules/local/generate_assembly_manifest/tests/main.nf.test index 7e9c3bf2..e16d278a 100644 --- a/modules/local/generate_assembly_manifest/tests/main.nf.test +++ b/modules/local/generate_assembly_manifest/tests/main.nf.test @@ -12,17 +12,16 @@ nextflow_process { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test' ], file(params.pipelines_testdata_base_path + "/test_data/assembly/contigs.fasta.gz", checkIfExists: true), file(params.pipelines_testdata_base_path + "/samplesheets/samplesheet_generatemanifest.csv", checkIfExists: true) - ] + ]) input[1] = "PRJ12345" input[2] = false input[3] = true input[4] = false """ - } } @@ -30,9 +29,6 @@ nextflow_process { assert process.success assertAll( // Only snapshot versions, not the manifest because it contains a timestamp - { assert snapshot( - path(process.out.versions[0]).yaml - ).match() }, { assert process.out.manifest.size() == 1 }, { assert process.out.manifest[0][1].toString().endsWith(".manifest") }, { @@ -76,8 +72,7 @@ nextflow_process { assert process.success assertAll( { assert snapshot( - process.out, - path(process.out.versions[0]).yaml + process.out ).match() } ) } diff --git a/modules/local/generate_assembly_manifest/tests/main.nf.test.snap b/modules/local/generate_assembly_manifest/tests/main.nf.test.snap index 7d4d1ef1..5a6e7664 100644 --- a/modules/local/generate_assembly_manifest/tests/main.nf.test.snap +++ b/modules/local/generate_assembly_manifest/tests/main.nf.test.snap @@ -11,7 +11,11 @@ ] ], "1": [ - "versions.yml:md5,5f33d074e4f7f7ddf4b3865973cb9157" + [ + "GENERATE_ASSEMBLY_MANIFEST", + "assembly_uploader", + "1.3.5" + ] ], "manifest": [ [ @@ -20,18 +24,10 @@ }, "test.manifest:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "versions": [ - "versions.yml:md5,5f33d074e4f7f7ddf4b3865973cb9157" ] - }, - { - "GENERATE_ASSEMBLY_MANIFEST": { - "assembly_uploader": "1.3.5" - } } ], - "timestamp": "2026-06-25T22:01:14.155809", + "timestamp": "2026-07-22T21:23:02.99938", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.0" diff --git a/modules/local/genome_upload/main.nf b/modules/local/genome_upload/main.nf index 651b281b..a0abb629 100644 --- a/modules/local/genome_upload/main.nf +++ b/modules/local/genome_upload/main.nf @@ -4,8 +4,8 @@ process GENOME_UPLOAD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/genome-uploader:3.0.1--pyhdfd78af_0': - 'quay.io/biocontainers/genome-uploader:3.0.1--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/genome-uploader:3.0.4--pyhdfd78af_0': + 'quay.io/biocontainers/genome-uploader:3.0.4--pyhdfd78af_0' }" input: path(mags) // required for validation @@ -22,7 +22,7 @@ process GENOME_UPLOAD { path "results/{MAG,bin}_upload/ENA_backup.json" , emit: ena_upload_backup_json path "results/{MAG,bin}_upload/genome_samples.xml" , emit: upload_genome_samples path "results/{MAG,bin}_upload/registered_{MAGs,bins}*.tsv", emit: upload_registered_mags - tuple val("${task.process}"), val('genome_uploader'), eval("genome_upload --version 2>&1 | sed 's/genome_uploader //g'"), topic: versions, emit: versions_genome_uploader + tuple val("${task.process}"), val('genome_uploader'), eval("genome_upload --version 2>&1 | sed 's/genome_uploader //g'"), topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/local/genome_upload/meta.yml b/modules/local/genome_upload/meta.yml index 14b54122..4364cf23 100644 --- a/modules/local/genome_upload/meta.yml +++ b/modules/local/genome_upload/meta.yml @@ -61,10 +61,6 @@ output: type: file description: XML file required for ENA submission pattern: "*submission.xml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@KateSakharova" diff --git a/modules/local/registerstudy/meta.yml b/modules/local/registerstudy/meta.yml index 803fc625..f4ca931d 100644 --- a/modules/local/registerstudy/meta.yml +++ b/modules/local/registerstudy/meta.yml @@ -59,11 +59,6 @@ output: JSON file containing the submission results with keys: submitted (newly created accessions) and failed. pattern: "*_accessions.json" - - versions: - - "versions.yml": - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@timrozday" diff --git a/modules/local/rename_fasta_for_catpack/main.nf b/modules/local/rename_fasta_for_catpack/main.nf index 38ac8569..e70cd15f 100644 --- a/modules/local/rename_fasta_for_catpack/main.nf +++ b/modules/local/rename_fasta_for_catpack/main.nf @@ -13,24 +13,32 @@ process RENAME_FASTA_FOR_CATPACK { tuple val(meta), path("output/*.fasta"), emit: renamed_fasta tuple val("${task.process}"), val('bash'), eval('bash --version | head -n1 | sed "s/.*version //; s/ .*//"'), topic: versions + when: + task.ext.when == null || task.ext.when script: - def is_compressed = fasta.name.endsWith('.gz') - extension = '.fasta' - def base = fasta.name - .replaceAll(/\.gz$/, '') - .replaceAll(/\.(fa|fasta|fna)$/, '') - def output_name = base + extension + def is_compressed = fasta.name.endsWith('.gz') + extension = '.fasta' + def base = fasta.name + .replaceAll(/\.gz$/, '') + .replaceAll(/\.(fa|fasta|fna)$/, '') + def output_name = base + extension - if (is_compressed) { - """ - mkdir -p output - gunzip -c ${fasta} > output/${output_name} - """ - } else { - """ - mkdir -p output - ln -s ../${fasta} output/${output_name} - """ - } + if (is_compressed) { + """ + mkdir -p output + gunzip -c ${fasta} > output/${output_name} + """ + } else { + """ + mkdir -p output + ln -s ../${fasta} output/${output_name} + """ + } + + stub: + """ + mkdir -p output + touch output/${meta.id}.fasta + """ } diff --git a/nextflow.config b/nextflow.config index eaad0b8c..d3911f83 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,7 +31,6 @@ params { // NCBI taxonomy cat_db = null - cat_db_download_id = 'nr' // MultiQC options multiqc_config = null @@ -243,7 +242,7 @@ profiles { test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_mag_complete_metadata { includeConfig 'conf/test_mag_complete_metadata.conf' } - test_mag_no_study_complete_metadata { includeConfig 'conf/test_mag_no_study_complete_metadata.config' } + test_mag_no_study_complete_metadata { includeConfig 'conf/test_mag_no_study_complete_metadata.config' } test_mag_no_coverage_single_reads { includeConfig 'conf/test_mag_no_coverage_single_reads.conf' } test_mag_no_coverage_paired_reads { includeConfig 'conf/test_mag_no_coverage_paired_reads.conf' } test_mag_no_quality { includeConfig 'conf/test_mag_no_quality.conf' } @@ -364,7 +363,7 @@ manifest { github: 'https://github.com/vagkaratzas', contribution: ['contributor'], // List of contribution types ('author', 'maintainer' or 'contributor') orcid: '0000-0001-9132-8981' - ], + ] ] homePage = 'https://github.com/nf-core/seqsubmit' description = """Submit sequences, assemblies, MAGs, etc., to public archives""" diff --git a/nextflow_schema.json b/nextflow_schema.json index 59c6dc02..bbd9f5d5 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -18,8 +18,8 @@ "exists": true, "mimetype": "text/csv", "pattern": "^\\S+\\.(csv|tsv|yaml|yml|json)$", - "description": "Path to comma-separated file describing the data to be submitted. Format depends on the pipeline mode (mags/bins/metagenomic_assemblies/reads).", - "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with a set of columns depending on the type of data being submitted. See [usage docs](https://nf-co.re/seqsubmit/usage#samplesheet-input).", + "description": "Path to samplesheet describing the data to be submitted (supported formats: csv, tsv, yaml, yml, json). Columns/fields depend on the pipeline mode", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a file in supported format with a set of columns depending on the type of data being submitted. See [usage docs](https://nf-co.re/seqsubmit/usage#samplesheet-input).", "fa_icon": "fas fa-file-csv" }, "outdir": { @@ -43,11 +43,11 @@ } }, "rna_detection_options": { - "title": "RNA detection options", + "title": "rRNA and tRNA detection options", "type": "object", "fa_icon": "fas fa-university", - "description": "Parameters used to make a decision about presence/absence of RNA in data", - "help_text": "ENA requires a boolean value (True/False) defining RNA. Decision of presence/absence of RNA is based on number of rRNA and tRNA. If both values are bigger than their corresponding limits - RNA is present (True), otherwise - not (False).", + "description": "Parameters affecting a decision about presence/absence of tRNA and rRNA genes (RNA_presence) in a genome", + "help_text": "The value of RNA_presence is based on the number of rRNA and tRNA. If at least 18 tRNAs are detected and 16S, 23S, and 5S rRNA subunits pass the recovery threshold - RNA_presence is True, otherwise - False. The resulting value is a decisive factor affecting MISAG/MIMAG assembly quality classification of a genome.", "properties": { "rrna_limit": { "type": "number", @@ -57,7 +57,7 @@ }, "trna_limit": { "type": "number", - "description": "Minimum number of tRNA.", + "description": "Minimum number of tRNA genes detected to count tRNA as present. This value is defined by the MISAG/MIMAG standard and must not be modified.", "default": 18, "fa_icon": "fas fa-users-cog" } @@ -71,17 +71,10 @@ "help_text": "Provide a path to pre-downloaded DB or DB id for automatic download", "properties": { "cat_db": { - "type": "string", + "format": "file-path", "description": "Path to local CAT_pack DB", - "help": "Database should be pre-downloaded using CATpack commands. The folder with database should contain sub-folders 'db' with files .dmnd, .fastaid2LCAtaxid, .taxids_with_multiple_offspring, etc. And subfolder 'tax' that should contain taxonomy files like .dmp, .taxdump.tar.gz, .prot.accession2taxid.FULL.gz, etc", + "help_text": "Database should be pre-downloaded using CATpack commands. The folder with database should contain sub-folders 'db' with files .dmnd, .fastaid2LCAtaxid, .taxids_with_multiple_offspring, etc, as well as subfolder 'tax' that should contain taxonomy files like .dmp, .taxdump.tar.gz, .prot.accession2taxid.FULL.gz, etc", "fa_icon": "fas fa-users-cog" - }, - "cat_db_download_id": { - "type": "string", - "description": "CAT_pack database ID for download", - "help": "There are two options available: nr (default) NCBI non-redundant protein database and gtdb (Genome Taxonomy Database (GTDB) proteins). We use NCBI proteins because ENA supports only NCBI format of taxonomy for submission", - "fa_icon": "fas fa-users-cog", - "default": "nr" } } }, @@ -93,13 +86,13 @@ "help_text": "These options are used to run Checkm2 that will calculate completeness and contamination of genome if those were not provided as input", "properties": { "checkm2_db": { - "type": "string", - "description": "Path to pre-downloaded checkm2 database", + "format": "file-path", + "description": "Path to pre-downloaded CheckM2 database. If omitted, downloads the version set by `--checkm2_db_download_id`", "fa_icon": "fas fa-users-cog" }, "checkm2_db_download_id": { "type": "integer", - "description": "Zenodo ID for checkm2 database download", + "description": "Zenodo ID for CheckM2 database download", "default": 14897628, "fa_icon": "fas fa-users-cog" } @@ -263,39 +256,39 @@ "properties": { "centre_name": { "type": "string", - "description": "The center name of the submitter (mandatory for broker accounts).", - "help": "ena-webin-cli argument -centerName: the center name of the submitter (mandatory for broker accounts)." + "description": "Name of the submitter's organisation (mandatory for broker accounts).", + "help_text": "ena-webin-cli argument -centerName: the center name of the submitter (mandatory for broker accounts)." }, "upload_tpa": { "type": "boolean", - "description": "Flag to control the type of assembly study (third party assembly or not)", - "default": false, - "help": "Use this flag if the study is a third party assembly (TPA). Default: false" + "description": "Mark the assemblies as a Third PArty (TPA)", + "help_text": "Set to true if the assembly was built from public sequence data (TPA) rather than original primary data. See NCBI TPA guidance: https://www.ncbi.nlm.nih.gov/genbank/tpa/", + "default": false }, "mode": { "type": "string", "default": null, - "description": "Type of upload", + "description": "Type of the data to be submitted", "help_text": "Different types of data require specific upload steps. That mode controls what upload workflow to run depending on type of data (mags/bins/metagenomic_assemblies/reads)", "enum": ["mags", "bins", "metagenomic_assemblies", "reads"] }, "test_upload": { "type": "boolean", - "description": "Upload to TEST server instead of LIVE", + "description": "Upload to ENA TEST server instead of LIVE server", "default": true, - "help": "Use that flag for development purposes or for validation of your submissions. Submission would be done to TEST ENA server instead of LIVE server. TEST server holds data 24h and removes afterwords. LIVE server does not have option to modify your submission, you can only suppress data and re-upload" + "help_text": "Use that flag for development purposes or for validation of your submissions. Submission would be done to TEST ENA server instead of LIVE server. TEST server holds data 24h and removes afterwords. LIVE server does not have option to modify your submission, you can only suppress data and re-upload" }, "is_private": { "type": "boolean", - "description": "Use that flag if you are referring to private data in ENA", "default": false, - "help": "This flag is used during metadata fetch steps. If you are referring to private data (project/reads/assembly) set to True. That flag does NOT regulate privacy of your submission data (check --release_date)" + "description": "Use that flag if you are referencing private data accessions in the samplesheet", + "help_text": "Set to true when the referenced project/reads/assembly are still private in ENA, so metadata is fetched via authenticated access. Does NOT affect the privacy of your own submission, use --release_date for that" }, "release_date": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$", - "description": "Use that flag if you want to keep your data private after submission", - "help": "This flag regulates privacy of created study for submission. If you want to keep your data private you need to specify a DATE when it can become public (YYYY-MM-DD, max 2 years from now)" + "description": "Date (YYYY-MM-DD) until which the newly created study stays private in ENA.", + "help_text": "Sets the release date for the study created by the pipeline. Data stays private until this date, then is automatically released and indexed by ENA. Must be no more than 2 years from today. Omit to release the study publicly immediately." }, "submission_study": { "type": "string", @@ -306,16 +299,16 @@ "type": "string", "format": "file-path", "exists": true, - "description": "Path to study metadata file (JSON, CSV, or TSV) for registering a new ENA study. Required when submission_study is not provided.", - "help_text": "File containing study metadata fields (required: study_title and alias, optional: study_abstract, existing_study_type, etc.). Used by REGISTERSTUDY to create a new study in ENA when no existing submission_study accession is given.", + "description": "Path to a file (JSON, CSV, or TSV) with metadata for registering a new ENA study. Required if --submission_study is not set.", + "help_text": "File containing study metadata fields (required: study_title, alias; optional: study_abstract, existing_study_type, etc.) used by the pipeline to register a new ENA study when no existing --submission_study accession is given. ENA docs: https://ena-docs.readthedocs.io/en/latest/submit/study/programmatic.html", "fa_icon": "fas fa-file-alt" }, "webincli_mode": { "type": "string", - "description": "Webin-CLI mode of operation", + "description": "Webin-CLI mode for ENA interaction: `submit` uploads data, `validate` performs validation only.", "default": "submit", "enum": ["submit", "validate"], - "help": "Controls ENA Webin-CLI mode. Use `submit` to upload data or `validate` to run validation only. Default: submit." + "help_text": "Controls ENA Webin-CLI mode. `validate` mode checks correctness of manifest files, data access and validity. `submit` mode does validation and push data to ENA server. Default: submit." } } } diff --git a/ro-crate-metadata.json b/ro-crate-metadata.json index d420ccc6..d27c3d25 100644 --- a/ro-crate-metadata.json +++ b/ro-crate-metadata.json @@ -21,9 +21,9 @@ { "@id": "./", "@type": "Dataset", - "creativeWorkStatus": "InProgress", - "datePublished": "2026-04-30T13:32:49+00:00", - "description": "

\n \n \n \"nf-core/seqsubmit\"\n \n

\n\n[![Open in GitHub Codespaces](https://img.shields.io/badge/Open_In_GitHub_Codespaces-black?labelColor=grey&logo=github)](https://github.com/codespaces/new/nf-core/seqsubmit)\n[![GitHub Actions CI Status](https://github.com/nf-core/seqsubmit/actions/workflows/nf-test.yml/badge.svg)](https://github.com/nf-core/seqsubmit/actions/workflows/nf-test.yml)\n[![GitHub Actions Linting Status](https://github.com/nf-core/seqsubmit/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/seqsubmit/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/seqsubmit/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX)\n[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)\n\n[![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.10.4-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)\n[![nf-core template version](https://img.shields.io/badge/nf--core_template-4.0.2-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/4.0.2)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/nf-core/seqsubmit)\n\n[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23seqsubmit-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/seqsubmit)[![Follow on Bluesky](https://img.shields.io/badge/bluesky-%40nf__core-1185fe?labelColor=000000&logo=bluesky)](https://bsky.app/profile/nf-co.re)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core)\n\n## Introduction\n\n**nf-core/seqsubmit** is a Nextflow pipeline for submitting sequence data to [ENA](https://www.ebi.ac.uk/ena/browser/home).\nCurrently, the pipeline supports four submission modes, each routed to a dedicated workflow and requiring its own input samplesheet structure:\n\n- `mags` for Metagenome Assembled Genomes (MAGs) submission with `GENOMESUBMIT` workflow\n- `bins` for bins submission with `GENOMESUBMIT` workflow\n- `metagenomic_assemblies` for assembly submission with `ASSEMBLYSUBMIT` workflow\n- `reads` for raw sequencing reads submission with `READSUBMIT` workflow\n\n![seqsubmit workflow diagram](assets/seqsubmit_schema.png)\n\n## Requirements\n\n- [Nextflow](https://www.nextflow.io/) `>=25.04.0`\n- Webin account registered at https://www.ebi.ac.uk/ena/submit/webin/login\n- Raw reads used to assemble contigs submitted to [INSDC](https://www.insdc.org/) and associated accessions available\n\nSetup your environment secrets before running the pipeline:\n\n`nextflow secrets set ENA_WEBIN \"Webin-XXX\"`\n\n`nextflow secrets set ENA_WEBIN_PASSWORD \"XXX\"`\n\nMake sure you update commands above with your authorised credentials.\n\n## Input samplesheets\n\nFor detailed descriptions of all samplesheet columns, see the [usage documentation](docs/usage.md#samplesheet-input).\n\n### `mags` and `bins` modes (`GENOMESUBMIT`)\n\nThe input must follow `assets/schema_input_genome.json`.\n\nRequired columns:\n\n- `sample`\n- `fasta` (must end with `.fa.gz`, `.fasta.gz`, or `.fna.gz`)\n- `accession`\n- `assembly_software`\n- `binning_software`\n- `binning_parameters`\n- `metagenome`\n- `environmental_medium`\n- `broad_environment`\n- `local_environment`\n- `co-assembly`\n\nAt least one of the following must be provided per row:\n\n- reads (`fastq_1`, optional `fastq_2` for paired-end)\n- `genome_coverage`\n\nAdditional supported columns:\n\n- `stats_generation_software`\n- `completeness`\n- `contamination`\n- `RNA_presence`\n- `NCBI_lineage`\n\nIf `genome_coverage`, `stats_generation_software`, `completeness`, `contamination`, `RNA_presence`, or `NCBI_lineage` are missing, the workflow can calculate or infer them when the required inputs are available.\n\nThose fields are metadata required for the [genome_uploader](https://github.com/EBI-Metagenomics/genome_uploader) package.\n\nExample `samplesheet_genomes.csv`:\n\n```csv\nsample,fasta,accession,fastq_1,fastq_2,assembly_software,binning_software,binning_parameters,stats_generation_software,completeness,contamination,genome_coverage,metagenome,co-assembly,broad_environment,local_environment,environmental_medium,RNA_presence,NCBI_lineage\nlachnospira_eligens,data/bin_lachnospira_eligens.fa.gz,SRR24458089,,,spades_v3.15.5,metabat2_v2.6,default,CheckM2_v1.0.1,61.0,0.21,32.07,sediment metagenome,No,marine,cable_bacteria,marine_sediment,No,d__Bacteria;p__Proteobacteria;s__unclassified_Proteobacteria\n```\n\n> [!IMPORTANT]\n> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown.\n\n### `metagenomic_assemblies` mode (`ASSEMBLYSUBMIT`)\n\nThe input must follow `assets/schema_input_assembly.json`.\n\nRequired columns:\n\n- `sample`\n- `fasta` (must end with `.fa.gz`, `.fasta.gz`, or `.fna.gz`)\n- `run_accession`\n- `assembler`\n- `assembler_version`\n\nAt least one of the following must be provided per row:\n\n- reads (`fastq_1`, optional `fastq_2` for paired-end)\n- `coverage`\n\nIf `coverage` is missing and reads are provided, the workflow calculates average coverage with `coverm`.\n\nExample `samplesheet_assembly.csv`:\n\n```csv\nsample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version\nassembly_1,data/contigs_1.fasta.gz,data/reads_1.fastq.gz,data/reads_2.fastq.gz,,ERR011322,SPAdes,3.15.5\nassembly_2,data/contigs_2.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9\n```\n\n> [!IMPORTANT]\n> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown.\n\n### `reads` mode (`READSUBMIT`)\n\nThe input must follow `assets/schema_input_reads.json`.\n\nRequired columns:\n\n- `sample`\n- `sample_accession`\n- `fastq_1`\n- `fastq_2`\n- `platform`\n- `instrument`\n- `library_source`\n- `library_selection`\n- `library_strategy`\n\nOptional columns:\n\n- `insert_size`\n- `library_name`\n- `description`\n\nExample `samplesheet_reads.csv`:\n\n```csv\nsample,sample_accession,fastq_1,fastq_2,platform,instrument,library_source,library_selection,library_strategy,insert_size,library_name,description\nillumina_run_001,SAMEA1234567,data/reads_R1.fastq.gz,data/reads_R2.fastq.gz,ILLUMINA,Illumina HiSeq 2000,GENOMIC,RANDOM,WGS,500,HiSeq_library_001,Illumina sequencing of sample XYZ\n```\n\n> [!IMPORTANT]\n> **Samplesheet column requirements**: All columns shown in the example above must be present in your samplesheet, even if some values are empty. Columns must be in exactly the same order as shown.\n\n## Usage\n\n> [!NOTE]\n> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/get_started/environment_setup/overview) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/get_started/run-your-first-pipeline) with `-profile test` before running the workflow on actual data.\n\n### Submission study\n\nAll data submitted through this pipeline must be associated with an ENA study (project). You can either pass an accession of your existing study via `--submission_study`or provide a metadata file via `--study_metadata` and the pipeline will register the study with ENA before submitting your data.\n\nSee the [usage documentation](docs/usage.md#submission-study) for more details.\n\n### Data privacy\n\nYou can reference private ENA data if you have access to it via your Webin account (it was submitted previously under your credentials). To use private data accessions in your submission metadata, specify the `--is_private` flag. The pipeline will then use your provided Webin credentials to fetch the required metadata.\n\nYou can also control the privacy of your submitted data:\n\n- If you provide an existing study via `--submission_study`, your submission will inherit the same privacy status as that study.\n- If no study is provided, the pipeline will register a new one for you. To keep this new study private, you must specify a release date using `--release_date YYYY-MM-DD` (up to 2 years from the current date).\n\n| Example | Source Data | Submission Visibility | Required Arguments | Result |\n| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------- | ------------------------------------------------------ | ----------------------------------------------------- |\n| Assemblies generated from public reads. Assemblies should be public immediately after submission | public | public | \u2013 | SUCCESS |\n| Assemblies generated from public reads. Assemblies should remain private for 1 year after submission | public | private | `--release_date YYYY-MM-DD` (+1 year) | SUCCESS |\n| Bins generated from private reads and assemblies. Bins should be public after submission. User **has access** to the reads and assemblies via Webin account | private | public | `--is_private` | SUCCESS |\n| Bins generated from private reads and assemblies. Bins should remain private for 2 years after submission. User **has access** via Webin account | private | private | `--is_private`, `--release_date YYYY-MM-DD` (+2 years) | SUCCESS |\n| MAGs generated from private reads and assemblies. MAGs should be public after submission. User **does not have access** to the reads and assemblies via Webin account | private | public | \u2013 | ERROR (submission can only reference accessible data) |\n\n### Database setup (`CheckM2` and `CAT_pack`)\n\nThe `mags`/`bins` workflow requires databases for completeness/contamination estimation and taxonomy assignment. See [Usage documentation](usage.md) for details.\n\n### Required parameters:\n\n| Parameter | Description |\n| ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------- |\n| `--mode` | Type of the data to be submitted. Options: `[mags, bins, metagenomic_assemblies, reads]` |\n| `--input` | Path to the samplesheet describing the data to be submitted |\n| `--outdir` | Path to the output directory for pipeline results |\n| `--submission_study` OR `--study_metadata` | ENA study accession (PRJ/ERP) to submit the data to OR metadata file in JSON/TSV/CSV format to register new study |\n| `--centre_name` | Name of the submitter's organisation |\n\n### Optional parameters:\n\n| Parameter | Description |\n| ----------------- | ------------------------------------------------------------------------------------------ |\n| `--upload_tpa` | Flag to control the type of assembly study (third party assembly or not). Default: false |\n| `--test_upload` | Upload to TEST ENA server instead of LIVE. Default: true |\n| `--webincli_mode` | Choose Webin-CLI mode: `submit` or `validate`. Default: `submit` |\n| `--private` | Use that flag if you are referring to private data in ENA |\n| `--release_date` | Use that flag if you want to keep your data private after submission until particular date |\n\nGeneral command template:\n\n```bash\nnextflow run nf-core/seqsubmit \\\n -profile \\\n --mode \\\n --input \\\n --centre_name \\\n --submission_study \\\n --outdir \n```\n\nTest run (submission to the ENA TEST server) in `mags` mode:\n\n```bash\nnextflow run nf-core/seqsubmit \\\n -profile docker \\\n --mode mags \\\n --input assets/samplesheet_genomes.csv \\\n --submission_study \\\n --centre_name TEST_CENTER \\\n --webincli_mode submit \\\n --test_upload true \\\n --outdir results/validate_mags\n```\n\nTest run (submission to the ENA TEST server) in `metagenomic_assemblies` mode:\n\n```bash\nnextflow run nf-core/seqsubmit \\\n -profile docker \\\n --mode metagenomic_assemblies \\\n --input assets/samplesheet_assembly.csv \\\n --submission_study \\\n --centre_name TEST_CENTER \\\n --webincli_mode submit \\\n --test_upload true \\\n --outdir results/validate_assemblies\n```\n\nLive submission example:\n\n```bash\nnextflow run nf-core/seqsubmit \\\n -profile docker \\\n --mode metagenomic_assemblies \\\n --input assets/samplesheet_assembly.csv \\\n --submission_study PRJEB98843 \\\n --test_upload false \\\n --webincli_mode submit \\\n --outdir results/live_assembly\n```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/running/run-pipelines#using-parameter-files).\n\nFor more details and further functionality, please refer to the [usage documentation](https://nf-co.re/seqsubmit/usage) and the [parameter documentation](https://nf-co.re/seqsubmit/parameters).\n\n## Pipeline output\n\nKey output locations in `--outdir`:\n\n- `mags/` or `bins/`: genome metadata, manifests, and per-sample submission support files\n- `metagenomic_assemblies/`: assembly metadata CSVs and per-sample coverage files\n- `multiqc/`: MultiQC summary report\n- `pipeline_info/`: execution reports, trace, DAG, and software versions\n\nFor full details, see the [output documentation](https://nf-co.re/seqsubmit/output).\n\n## Credits\n\nnf-core/seqsubmit was originally written by [Martin Beracochea](https://github.com/mberacochea), [Ekaterina Sakharova](https://github.com/KateSakharova), [Sofia Ochkalova](https://github.com/ochkalova), [Evangelos Karatzas](https://github.com/vagkaratzas) and [Tim Rozday](https://github.com/timrozday-mgnify).\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](docs/CONTRIBUTING.md).\n\nFor further information or help, don't hesitate to get in touch on the [Slack `#seqsubmit` channel](https://nfcore.slack.com/channels/seqsubmit) (you can join with [this invite](https://nf-co.re/join/slack)).\n\n## Citations\n\n\n\n\n\nIf you use this pipeline please make sure to cite all used software.\nThis pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/main/LICENSE).\n\n> **MGnify: the microbiome sequence data analysis resource in 2023**\n>\n> Richardson L, Allen B, Baldi G, Beracochea M, Bileschi ML, Burdett T, et al.\n>\n> Vol. 51, Nucleic Acids Research. Oxford University Press (OUP); 2022. p. D753\u20139. Available from: http://dx.doi.org/10.1093/nar/gkac1080\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nYou can cite the `nf-core` publication as follows:\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", + "creativeWorkStatus": "Stable", + "datePublished": "2026-07-24T12:36:22+00:00", + "description": "

\n \n \n \"nf-core/seqsubmit\"\n \n

\n\n[![Open in GitHub Codespaces](https://img.shields.io/badge/Open_In_GitHub_Codespaces-black?labelColor=grey&logo=github)](https://github.com/codespaces/new/nf-core/seqsubmit)\n[![GitHub Actions CI Status](https://github.com/nf-core/seqsubmit/actions/workflows/nf-test.yml/badge.svg)](https://github.com/nf-core/seqsubmit/actions/workflows/nf-test.yml)\n[![GitHub Actions Linting Status](https://github.com/nf-core/seqsubmit/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/seqsubmit/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/seqsubmit/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX)\n[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)\n\n[![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.10.4-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)\n[![nf-core template version](https://img.shields.io/badge/nf--core_template-4.0.3-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/4.0.3)\n[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)\n[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)\n[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)\n[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/nf-core/seqsubmit)\n\n[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23seqsubmit-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/seqsubmit)[![Follow on Bluesky](https://img.shields.io/badge/bluesky-%40nf__core-1185fe?labelColor=000000&logo=bluesky)](https://bsky.app/profile/nf-co.re)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core)\n\n## Introduction\n\n**nf-core/seqsubmit** is a bioinformatics pipeline that ...\n\n\n\n\n2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/))\n\n## Usage\n\n> [!NOTE]\n> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/get_started/environment_setup/overview) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/get_started/run-your-first-pipeline) with `-profile test` before running the workflow on actual data.\n\n\n\nNow, you can run the pipeline using:\n\n\n\n```bash\nnextflow run nf-core/seqsubmit \\\n -profile \\\n --input samplesheet.csv \\\n --outdir \n```\n\n> [!WARNING]\n> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/running/run-pipelines#using-parameter-files).\n\nFor more details and further functionality, please refer to the [usage documentation](https://nf-co.re/seqsubmit/usage) and the [parameter documentation](https://nf-co.re/seqsubmit/parameters).\n\n## Pipeline output\n\nTo see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/seqsubmit/results) tab on the nf-core website pipeline page.\nFor more details about the output files and reports, please refer to the\n[output documentation](https://nf-co.re/seqsubmit/output).\n\n## Credits\n\nnf-core/seqsubmit was originally written by Martin Beracochea, Ekaterina Sakharova, Sofia Ochkalova, Evangelos Karatzas, Tim Rozday.\n\nWe thank the following people for their extensive assistance in the development of this pipeline:\n\n\n\n## Contributions and Support\n\nIf you would like to contribute to this pipeline, please see the [contributing guidelines](docs/CONTRIBUTING.md).\n\nFor further information or help, don't hesitate to get in touch on the [Slack `#seqsubmit` channel](https://nfcore.slack.com/channels/seqsubmit) (you can join with [this invite](https://nf-co.re/join/slack)).\n\n## Citations\n\n\n\n\n\n\nAn extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.\n\nYou can cite the `nf-core` publication as follows:\n\n> **The nf-core framework for community-curated bioinformatics pipelines.**\n>\n> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen.\n>\n> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x).\n", "hasPart": [ { "@id": "main.nf" @@ -102,7 +102,7 @@ }, "mentions": [ { - "@id": "#6a35cf24-1f54-4f1b-8bd7-da24e4ef9c61" + "@id": "#b9a22430-cd92-4053-a257-9fd6f20776f9" } ], "name": "nf-core/seqsubmit" @@ -134,17 +134,20 @@ "@id": "https://orcid.org/0000-0003-3472-3736" }, { - "@id": "#002b989f-efa9-4c5b-bf4e-bbf8f2d58be1" + "@id": "#e5eaba7e-b8d4-40dd-a235-ea4a18314307" }, { - "@id": "#bf7a74fc-7f56-4751-bd51-0f3035468776" + "@id": "https://orcid.org/0000-0003-2814-6012" }, { "@id": "https://orcid.org/0000-0001-9132-8981" + }, + { + "@id": "#14ef52aa-83db-4988-82ab-828335934408" } ], "dateCreated": "", - "dateModified": "2026-04-30T13:32:49Z", + "dateModified": "2026-07-24T12:36:22Z", "dct:conformsTo": "https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE/", "keywords": [ "nf-core", @@ -162,13 +165,8 @@ "sdPublisher": { "@id": "https://nf-co.re/" }, - "url": [ - "https://github.com/nf-core/seqsubmit", - "https://nf-co.re/seqsubmit/1.0.0/" - ], - "version": [ - "1.0.0" - ] + "url": ["https://github.com/nf-core/seqsubmit", "https://nf-co.re/seqsubmit/1.0.0/"], + "version": ["1.0.0"] }, { "@id": "https://w3id.org/workflowhub/workflow-ro-crate#nextflow", @@ -183,11 +181,11 @@ "version": "!>=25.10.4" }, { - "@id": "#6a35cf24-1f54-4f1b-8bd7-da24e4ef9c61", + "@id": "#b9a22430-cd92-4053-a257-9fd6f20776f9", "@type": "TestSuite", "instance": [ { - "@id": "#32731c67-24d2-422c-bacc-c9bf3b9753c3" + "@id": "#4b475711-84b9-4a69-9fa5-544aba88ac60" } ], "mainEntity": { @@ -196,7 +194,7 @@ "name": "Test suite for nf-core/seqsubmit" }, { - "@id": "#32731c67-24d2-422c-bacc-c9bf3b9753c3", + "@id": "#4b475711-84b9-4a69-9fa5-544aba88ac60", "@type": "TestInstance", "name": "GitHub Actions workflow for testing nf-core/seqsubmit", "resource": "repos/nf-core/seqsubmit/actions/workflows/nf-test.yml", @@ -335,12 +333,12 @@ "name": "Martin Beracochea" }, { - "@id": "#002b989f-efa9-4c5b-bf4e-bbf8f2d58be1", + "@id": "#e5eaba7e-b8d4-40dd-a235-ea4a18314307", "@type": "Person", "name": "Ekaterina Sakharova" }, { - "@id": "#bf7a74fc-7f56-4751-bd51-0f3035468776", + "@id": "https://orcid.org/0000-0003-2814-6012", "@type": "Person", "name": "Sofia Ochkalova" }, @@ -348,6 +346,11 @@ "@id": "https://orcid.org/0000-0001-9132-8981", "@type": "Person", "name": "Evangelos Karatzas" + }, + { + "@id": "#14ef52aa-83db-4988-82ab-828335934408", + "@type": "Person", + "name": "Tim Rozday" } ] } \ No newline at end of file diff --git a/subworkflows/local/fasta_validation/main.nf b/subworkflows/local/fasta_validation/main.nf index 4d2d5599..c59df839 100644 --- a/subworkflows/local/fasta_validation/main.nf +++ b/subworkflows/local/fasta_validation/main.nf @@ -8,7 +8,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { FALINT } from '../../../modules/nf-core/falint/main' +include { FALINT } from '../../../modules/nf-core/falint/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -63,5 +63,5 @@ workflow FASTA_VALIDATION { ) emit: - valid_fastas = valid_fastas + valid_fastas = valid_fastas // [meta, fasta_file] } diff --git a/subworkflows/local/genome_evaluation/main.nf b/subworkflows/local/genome_evaluation/main.nf index 758f903b..8b74a36a 100644 --- a/subworkflows/local/genome_evaluation/main.nf +++ b/subworkflows/local/genome_evaluation/main.nf @@ -20,32 +20,26 @@ include { CHECKM2_PREDICT } from '../../../modules/nf-core/checkm2/pred workflow GENOME_EVALUATION { take: - ch_fasta // channel: [ val(meta), path(fasta) ] - ch_checkm2_db // channel: [ val(meta), path(db) ] - pre-built db as directory - // provide channel.empty() to trigger automatic download via ch_checkm2_db_download_id - ch_checkm2_db_download_id // channel: [ val(meta), val(db_id) ] - db ID for CHECKM2_DATABASEDOWNLOAD (e.g. '1234567') - // only used if ch_checkm2_db is empty + ch_fasta // channel: [ val(meta), path(fasta) ] + ch_checkm2_db // val: path to local CheckM2 database or null to trigger automatic download + ch_checkm2_db_download_id // val: CheckM2 database Zenodo download ID main: - ch_versions = channel.empty() // // Database preparation // + ch_do_download = ch_fasta + .first() + .map { _meta_fasta -> ch_checkm2_db_download_id } // Download and prepare db from scratch if no pre-built db provided - // Only trigger if ch_fasta has items - ch_download_trigger = ch_checkm2_db - .count() - .filter { count -> count == 0 } // Only proceed if ch_checkm2_db is empty - .combine(ch_fasta.first()) - .combine(ch_checkm2_db_download_id) - .map { _count, _meta, _fasta, db_meta, db_id -> [db_meta, db_id] } - - CHECKM2_DATABASEDOWNLOAD(ch_download_trigger) - - // Combine db sources - one of these channels will be empty depending on inputs - ch_db = ch_checkm2_db.mix(CHECKM2_DATABASEDOWNLOAD.out.database).first() + if (ch_checkm2_db) { + ch_checkm2_db_ready = channel.value([[id: 'checkm2_db'], file(params.checkm2_db, checkIfExists: true)]) + } else { + CHECKM2_DATABASEDOWNLOAD(ch_do_download) + ch_checkm2_db_ready = CHECKM2_DATABASEDOWNLOAD.out.database + } // // Genome evaluation @@ -53,7 +47,7 @@ workflow GENOME_EVALUATION { CHECKM2_PREDICT( ch_fasta, - ch_db, + ch_checkm2_db_ready, ) emit: diff --git a/subworkflows/local/rna_detection/main.nf b/subworkflows/local/rna_detection/main.nf index bb0a23fa..0cac5bb8 100644 --- a/subworkflows/local/rna_detection/main.nf +++ b/subworkflows/local/rna_detection/main.nf @@ -26,7 +26,6 @@ workflow RNA_DETECTION { main: - ch_versions = channel.empty() BARRNAP( fasta.map {id, fasta_file -> [id, fasta_file, "bac"]} ) @@ -43,5 +42,4 @@ workflow RNA_DETECTION { emit: rna_detected = COUNT_RNA.out.rna_decision - versions = ch_versions } diff --git a/subworkflows/local/rna_detection/meta.yml b/subworkflows/local/rna_detection/meta.yml index 4650e921..84902bab 100644 --- a/subworkflows/local/rna_detection/meta.yml +++ b/subworkflows/local/rna_detection/meta.yml @@ -43,11 +43,6 @@ output: pattern: "*.tsv" ontologies: - edam: "http://edamontology.org/format_3475" # TSV - - versions: - type: file - description: | - Channel containing software versions. - Structure: versions authors: - "@KateSakharova" - "@ochkalova" diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf index 1ac05e90..c6dea444 100644 --- a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf @@ -35,6 +35,8 @@ workflow PIPELINE_INITIALISATION { help // boolean: Display help message and exit help_full // boolean: Show the full help message show_hidden // boolean: Show hidden parameters in the help message + submission_study // val: ENA study accession for all reads in this run (optional). + study_metadata // val: path to study metadata file for study creation (used if no submission_study provided) main: @@ -71,7 +73,7 @@ workflow PIPELINE_INITIALISATION { https://doi.org/10.1038/s41587-020-0439-x * Software dependencies - https://github.com/nf-core/seqsubmit/blob/master/CITATIONS.md + https://github.com/nf-core/seqsubmit/blob/main/CITATIONS.md """ if (monochrome_logs) { before_text = before_text.replaceAll(/\033\[[0-9;]*m/, '') @@ -112,8 +114,11 @@ workflow PIPELINE_INITIALISATION { } else if ( mode == "reads" ) { ch_samplesheet = channel .fromList(samplesheetToList(input, "${projectDir}/assets/schema_input_reads.json")) - } else { - error("Unknown mode specified: '${mode}'. Supported modes are 'mags', 'bins', 'metagenomic_assemblies', and 'reads'.") + } + + // check submission study and metadata presence/absence for REGISTER STUDY + if (!submission_study && !study_metadata) { + error("Either --submission_study or --study_metadata must be provided") } emit: @@ -200,7 +205,7 @@ def toolCitationText() { ].join(' ').trim() def preprocessing_tools = [ - "Input FASTA validation was done by py_fasta_validator (Edwards et al. 2023)." + "Input FASTA validation was done by fa-lint (https://github.com/GallVp/fa-lint)." ].join(' ').trim() def stats_tools = [ @@ -237,12 +242,12 @@ def toolBibliographyText() { def reference_text = [ '
  • Richardson, L., Allen, B., Baldi, G., Beracochea, M., Bileschi, M. L., Burdett, T., ... & Finn, R. D. (2023). MGnify: the microbiome sequence data analysis resource in 2023. Nucleic acids research, 51(D1), D753-D759. doi: 10.1093/nar/gkac1080
    • ', '
  • Gurbich, T. A., Almeida, A., Beracochea, M., Burdett, T., Burgin, J., Cochrane, G., ... & Finn, R. D. (2023). MGnify genomes: a resource for biome-specific microbial genome catalogues. Journal of molecular biology, 435(14), 168016. doi: 10.1016/j.jmb.2023.168016
    • ', - '
  • webin-cli: David, Y., Alisha, A., Awais, A., Rajkumar, D., Dipayan, G., Muhammad, H., ... & Ugis, S. (2026). The European Nucleotide Archive in 2025. Nucleic Acids Research, 54(D1), D120-D127. doi: 10.1093/nar/gkaf1295
    • ', + '
  • David, Y., Alisha, A., Awais, A., Rajkumar, D., Dipayan, G., Muhammad, H., ... & Ugis, S. (2026). The European Nucleotide Archive in 2025. Nucleic Acids Research, 54(D1), D120-D127. doi: 10.1093/nar/gkaf1295
    • ', '
  • Edwards, R. (2023). linsalrob/py_fasta_validator: Compressy. doi: 10.5281/zenodo.5002710
    • ', '
  • Chklovski, A., Parks, D. H., Woodcroft, B. J., & Tyson, G. W. (2023). CheckM2: a rapid, scalable and accurate tool for assessing microbial genome quality using machine learning. Nature Methods, 20(8), 1203-1212. doi: 10.1038/s41592-023-01940-w
    • ', '
  • Aroney, S. T., Newell, R. J., Nissen, J. N., Camargo, A. P., Tyson, G. W., & Woodcroft, B. J. (2025). CoverM: read alignment statistics for metagenomics. Bioinformatics, 41(4), btaf147. doi: 10.1093/bioinformatics/btaf147
    • ', '
  • Von Meijenfeldt, F. B., Arkhipova, K., Cambuy, D. D., Coutinho, F. H., & Dutilh, B. E. (2019). Robust taxonomic classification of uncharted microbial sequences and bins with CAT and BAT. Genome biology, 20(1), 217. doi: 10.1038/s41467-024-47155-1
    • ', - '
  • barrnap: Seemann, T. (2014). barrnap: Bacterial ribosomal RNA predictor. barrnap: Bacterial ribosomal RNA predictor. vicbioinformatics.com
    • ', + '
  • Seemann, T. (2014). barrnap: Bacterial ribosomal RNA predictor. barrnap: Bacterial ribosomal RNA predictor. vicbioinformatics.com
    • ', '
  • Chan, P. P., Lin, B. Y., Mak, A. J., & Lowe, T. M. (2021). tRNAscan-SE 2.0: improved detection and functional classification of transfer RNA genes. Nucleic acids research, 49(16), 9077-9096. doi: 10.1093/nar/gkab688
    • ', '
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: 10.1093/bioinformatics/btw354
    • ' ].join(' ').trim() diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 9968b79f..4aadca31 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -35,7 +35,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_complete_metadata.nf.test.snap b/tests/mag_complete_metadata.nf.test.snap index 7c7f539c..54957593 100644 --- a/tests/mag_complete_metadata.nf.test.snap +++ b/tests/mag_complete_metadata.nf.test.snap @@ -17,7 +17,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_no_coverage_paired_reads.nf.test.snap b/tests/mag_no_coverage_paired_reads.nf.test.snap index 894e7561..0725d600 100644 --- a/tests/mag_no_coverage_paired_reads.nf.test.snap +++ b/tests/mag_no_coverage_paired_reads.nf.test.snap @@ -20,7 +20,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_no_coverage_single_reads.nf.test.snap b/tests/mag_no_coverage_single_reads.nf.test.snap index 767748fa..3a432618 100644 --- a/tests/mag_no_coverage_single_reads.nf.test.snap +++ b/tests/mag_no_coverage_single_reads.nf.test.snap @@ -20,7 +20,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_no_rna_presence.nf.test.snap b/tests/mag_no_rna_presence.nf.test.snap index 8a7d21c7..55419566 100644 --- a/tests/mag_no_rna_presence.nf.test.snap +++ b/tests/mag_no_rna_presence.nf.test.snap @@ -23,7 +23,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_no_study_complete_metadata.nf.test.snap b/tests/mag_no_study_complete_metadata.nf.test.snap index 1aea1e8c..b838f6b5 100644 --- a/tests/mag_no_study_complete_metadata.nf.test.snap +++ b/tests/mag_no_study_complete_metadata.nf.test.snap @@ -17,7 +17,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/tests/mag_no_taxonomy.nf.test.snap b/tests/mag_no_taxonomy.nf.test.snap index 1563d4b4..0aa4bc4d 100644 --- a/tests/mag_no_taxonomy.nf.test.snap +++ b/tests/mag_no_taxonomy.nf.test.snap @@ -26,7 +26,7 @@ "bash": "5.2.37(1)-release" }, "CREATE_MANIFESTS": { - "genome_uploader": "3.0.1" + "genome_uploader": "3.0.4" }, "FALINT": { "falint": "1.2.0" diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 84a9dc81..90273abd 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -48,39 +48,22 @@ workflow ASSEMBLYSUBMIT { def ch_versions = channel.empty() def ch_multiqc_files = channel.empty() - // Create assembly channel with proper metadata structure + // --------- Create assembly channel with proper metadata structure assembly_fasta = ch_samplesheet - .map { row -> - def meta = [ - id: row[0].id, - single_end: row[3] ? false : true, - coverage: row[4] ?: null, - run_accession: row[5], - assembler: row[6], - assembler_version: row[7] - ] - [meta, file(row[1])] + .map { meta, fasta, reads_1, reads_2 -> + def new_meta = meta + [single_end: !reads_2] + [new_meta, fasta] } + // --------- Create reads channel with proper metadata structure reads_fastq = ch_samplesheet .filter { row -> row[2] && row[2] != "" } // Check if fastq_1 exists and is not empty - .map { row -> - def meta = [ - id: row[0].id, - single_end: row[3] ? false : true, - coverage: row[4] ?: null, - run_accession: row[5], - assembler: row[6], - assembler_version: row[7] - ] - - if (row[3] && row[3] != "") { - // If paired end reads - [meta, [file(row[2]), file(row[3])]] - } else { - // If single end - [meta, file(row[2])] - } + .map { meta, fasta, reads_1, reads_2 -> + def new_meta = meta + [single_end: !reads_2] + def reads = new_meta.single_end + ? [reads_1] + : [reads_1, reads_2] + [new_meta, reads] } // --------- Check fasta files are properly formatted and filter out files with less than 2 contigs @@ -88,15 +71,15 @@ workflow ASSEMBLYSUBMIT { assembly_fasta ) + // --------- Assembly coverage calculation // For assemblies without coverage, calculate coverage with CoverM - FASTA_VALIDATION.out.valid_fastas - .filter { meta, _fasta -> meta.coverage == null } + coverm_input = FASTA_VALIDATION.out.valid_fastas + .filter { meta, _fasta -> !(meta.coverage) } .join(reads_fastq) .multiMap { meta, fasta, fastq -> assembly: [ meta, fasta ] reads: [ meta, fastq ] } - .set { coverm_input } COVERM_CONTIG ( coverm_input.reads, @@ -119,7 +102,7 @@ workflow ASSEMBLYSUBMIT { } // Update metadata with calculated coverage - FASTA_VALIDATION.out.valid_fastas + assemblies_with_added_cov_ch = FASTA_VALIDATION.out.valid_fastas .filter { meta, _fasta -> meta.coverage == null } .join( average_coverage_ch ) .map { meta, fasta, avg_coverage -> @@ -127,7 +110,6 @@ workflow ASSEMBLYSUBMIT { updated_meta.coverage = avg_coverage [updated_meta, fasta] } - .set { assemblies_with_added_cov_ch } // Combine assemblies with updated metadata (for samples that had coverage calculated) // and assemblies that already had coverage @@ -135,20 +117,18 @@ workflow ASSEMBLYSUBMIT { .filter { meta, _fasta -> meta.coverage != null } .mix( assemblies_with_added_cov_ch ) - // Create CSV with assembly metadata for manifest generation + // --------- Create CSV with assembly metadata for manifest generation CREATE_ASSEMBLY_METADATA_CSV( assemblies_with_coverage ) - // Concatenate assembly metadata CSVs into single file to publish + // --------- Concatenate assembly metadata CSVs into single file to publish CONCAT_METADATA ( CREATE_ASSEMBLY_METADATA_CSV.out.csv.map { _meta, file -> file }.collect().map { files -> [ [id: "assemblies_metadata"], files ] }, 'true' // skip_header - we want to keep the header from the first file and skip it for the rest ) - if (!submission_study && !study_metadata) { - error("Either --submission_study or --study_metadata must be provided") - } + // --------- Register study if accession was not provided via --submission_study def study_accession_ch if (submission_study) { // Use provided study accession directly @@ -168,7 +148,7 @@ workflow ASSEMBLYSUBMIT { } } - // Generate assembly manifest files and submit them to ENA + // --------- Generate assembly manifest files and submit them to ENA GENERATE_ASSEMBLY_MANIFEST( assemblies_with_coverage.join(CREATE_ASSEMBLY_METADATA_CSV.out.csv), study_accession_ch.first(), @@ -176,8 +156,8 @@ workflow ASSEMBLYSUBMIT { test_upload, is_private ) - ch_versions = ch_versions.mix(GENERATE_ASSEMBLY_MANIFEST.out.versions.first()) + // --------- Upload data to ENA SUBMIT ( assemblies_with_coverage.join(GENERATE_ASSEMBLY_MANIFEST.out.manifest), test_upload, @@ -191,9 +171,7 @@ workflow ASSEMBLYSUBMIT { 'true' // skip_header - we want to keep the header from the first file and skip it for the rest ) - // - // Collate and save software versions - // + // --------- Collate and save software versions def topic_versions = channel.topic("versions") .distinct() .branch { entry -> @@ -220,9 +198,7 @@ workflow ASSEMBLYSUBMIT { newLine: true ) - // - // MODULE: MultiQC - // + // --------- MODULE: MultiQC ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) def ch_summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") def ch_workflow_summary = channel.value(paramsSummaryMultiqc(ch_summary_params)) @@ -232,8 +208,8 @@ workflow ASSEMBLYSUBMIT { : file("${projectDir}/assets/methods_description_template.yml", checkIfExists: true) def ch_methods_description = channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: true)) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{meta, file -> file}) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{_meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{_meta, file -> file}) MULTIQC( ch_multiqc_files.flatten().collect().map { files -> [ diff --git a/workflows/genomesubmit.nf b/workflows/genomesubmit.nf index 8c1df8e9..37ce9555 100644 --- a/workflows/genomesubmit.nf +++ b/workflows/genomesubmit.nf @@ -46,7 +46,6 @@ workflow GENOMESUBMIT { checkm2_db // val: path to CheckM2 database checkm2_db_download_id // val: CheckM2 database download ID cat_db // val: path to CAT database - cat_db_download_id // val: CAT database download ID centre_name // val: submission centre name upload_tpa // val: upload as TPA (Third Party Annotation) test_upload // val: true for test upload mode @@ -59,38 +58,14 @@ workflow GENOMESUBMIT { def ch_versions = channel.empty() def ch_multiqc_files = channel.empty() - // --------- Create genomes channel with proper metadata structure + // --------- Create genomes channel with proper metadata structure genome_fasta_and_reads = ch_samplesheet - .map { row -> - def meta = [ - id: row[0].id, - accession: row[2], - single_end: row[4] ? false : true, - assembly_software: row[5] ?: null, - binning_software: row[6] ?: null, - binning_parameters: row[7] ?: null, - stats_generation_software: row[8] ?: null, - completeness: row[9] ?: null, - contamination: row[10] ?: null, - genome_coverage: row[11] ?: null, - metagenome: row[12] ?: null, - co_assembly: row[13] ?: null, - broad_environment: row[14] ?: null, - local_environment: row[15] ?: null, - environmental_medium: row[16] ?: null, - RNA_presence: row[17] ?: null, - NCBI_lineage: row[18] ?: null - ] - def read1 = row[3] ? file(row[3]) : null - def read2 = row[4] ? file(row[4]) : null - - if (row[4] && row[4] != "") { - // If paired end reads - return [meta, file(row[1]), [read1, read2]] - } else { - // If single end - return [meta, file(row[1]), [read1]] - } + .map { meta, fasta, reads_1, reads_2 -> + def new_meta = meta + [single_end: !reads_2] + def reads = new_meta.single_end + ? [reads_1] + : [reads_1, reads_2] + [new_meta, fasta, reads] } genome_fasta = genome_fasta_and_reads.map{meta, fasta, _fq1 -> [meta, fasta]} @@ -102,19 +77,17 @@ workflow GENOMESUBMIT { ) // --------- Genome coverage calculation - FASTA_VALIDATION.out.valid_fastas + branched_coverage_results = FASTA_VALIDATION.out.valid_fastas .branch { meta, _fasta -> - genome_coverage_ref_input: meta.genome_coverage == null + genome_coverage_ref_input: !(meta.genome_coverage) genome_coverage_present: true // Everything else goes here } - .set { branched_coverage_results } - branched_coverage_results.genome_coverage_ref_input.join(genome_reads) + coverm_input = branched_coverage_results.genome_coverage_ref_input.join(genome_reads) .multiMap { meta, fasta, fastq -> genome: [ meta, fasta ] raw_reads: [ meta, fastq ] } - .set { coverm_input } COVERM_GENOME ( coverm_input.raw_reads, @@ -138,26 +111,24 @@ workflow GENOMESUBMIT { .mix(branched_coverage_results.genome_coverage_present) // --------- For genomes without RNA_presence info, calculate rRNA and tRNA - fasta_updated_with_coverage + branched_rna_results = fasta_updated_with_coverage .branch { meta, _fasta -> - rna_prediction_input: meta.RNA_presence == null + rna_prediction_input: meta.RNA_presence == null // it might be True/False rna_present: true // Everything else goes here } - .set { branched_rna_results } RNA_DETECTION ( branched_rna_results.rna_prediction_input, trna_limit, rrna_limit ) - ch_versions = ch_versions.mix( RNA_DETECTION.out.versions ) // Update metadata for records missing RNA fasta_updated_with_rna = RNA_DETECTION.out.rna_detected.join(branched_rna_results.rna_prediction_input) .map{ meta, rna_decision, fasta -> def lines = rna_decision.readLines() // support for empty decision files required for -stub mode - def decision = lines ? lines[0].split('\t')[1] : null + def decision = lines ? lines[0].split('\t')[1].toLowerCase() == 'true' : null def updated_meta = meta.clone() updated_meta.RNA_presence = decision return [updated_meta, fasta] @@ -165,26 +136,16 @@ workflow GENOMESUBMIT { .mix(branched_rna_results.rna_present) // --------- Completeness and contamination calculation - fasta_updated_with_rna + branched_stats_results = fasta_updated_with_rna .branch { meta, _fasta -> - genome_evaluation_input: meta.completeness == null || meta.contamination == null || meta.stats_generation_software == null + genome_evaluation_input: !(meta.completeness) || !(meta.contamination) || !(meta.stats_generation_software) evaluation_present: true // Everything else goes here } - .set { branched_stats_results } - - // build input structures for CheckM2 DB depending on what provided as input - def checkm2_db_input = checkm2_db - ? channel.of( [['id': 'CHECKM2_DB'], file(checkm2_db)] ) - : channel.empty() - - def checkm2_db_id_input = (!checkm2_db && checkm2_db_download_id) - ? channel.of( [['id': 'CHECKM2_DB_id'], checkm2_db_download_id] ) - : channel.empty() GENOME_EVALUATION ( branched_stats_results.genome_evaluation_input, - checkm2_db_input, - checkm2_db_id_input + checkm2_db, + checkm2_db_download_id ) // Create a value channel with the version string @@ -207,12 +168,11 @@ workflow GENOMESUBMIT { .mix(branched_stats_results.evaluation_present) // --------- Taxonomy - fasta_updated_with_stats + branched_taxonomy_results = fasta_updated_with_stats .branch { meta, _fasta -> - genome_taxonomy_input: meta.NCBI_lineage == null + genome_taxonomy_input: !(meta.NCBI_lineage) taxonomy_present: true // Everything else goes here } - .set { branched_taxonomy_results } // Change extension for all files required taxonomy to .fasta because CATPACK requires suffix as input RENAME_FASTA_FOR_CATPACK ( @@ -224,9 +184,10 @@ workflow GENOMESUBMIT { ? channel.of( [['id': 'CAT_DB'], file(cat_db)] ) : channel.empty() - def cat_db_id_input = (!cat_db && cat_db_download_id) - ? channel.of( [['id': 'CAT_DB_id'], cat_db_download_id] ) - : channel.empty() + cat_db_download_id = "nr" // NCBI non-redundant protein database identifier + def cat_db_id_input = cat_db + ? channel.empty() + : channel.of( [['id': 'CAT_DB_id'], cat_db_download_id] ) FASTA_CLASSIFY_CATPACK ( RENAME_FASTA_FOR_CATPACK.out.renamed_fasta, // ch_bins @@ -259,10 +220,7 @@ workflow GENOMESUBMIT { 'true' // skip_header - we want to keep the header from the first file and skip it for the rest ) - // --------- Register study if accession not provided - if (!submission_study && !study_metadata) { - error("Either --submission_study or --study_metadata must be provided") - } + // --------- Register study if accession not provided via --submission_study def study_accession_ch if (submission_study) { study_accession_ch = channel.of(submission_study) @@ -304,16 +262,15 @@ workflow GENOMESUBMIT { } // Combine fasta and manifests ch_combined = fasta_updated_with_taxonomy - .map { meta, fasta -> [meta.id, meta, fasta] } - .join( - manifests_ch.map { meta, manifest -> [meta.id, manifest] } // Has only [id: prefix] - ) - .map { _id, full_meta, fasta, manifest -> - [full_meta, fasta, manifest] - } + .map { meta, fasta -> [meta.id, meta, fasta] } + .join( + manifests_ch.map { meta, manifest -> [meta.id, manifest] } // Has only [id: prefix] + ) + .map { _id, full_meta, fasta, manifest -> + [full_meta, fasta, manifest] + } // --------- Upload data to ENA - SUBMIT ( ch_combined, test_upload, @@ -327,9 +284,7 @@ workflow GENOMESUBMIT { 'true' // skip_header - we want to keep the header from the first file and skip it for the rest ) - // - // Collate and save software versions - // + // --------- Collate and save software versions def topic_versions = channel.topic("versions") .distinct() .branch { entry -> @@ -356,10 +311,7 @@ workflow GENOMESUBMIT { newLine: true ) - - // - // MODULE: MultiQC - // + // --------- MODULE: MultiQC ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) def ch_summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") def ch_workflow_summary = channel.value(paramsSummaryMultiqc(ch_summary_params)) @@ -369,9 +321,9 @@ workflow GENOMESUBMIT { : file("${projectDir}/assets/methods_description_template.yml", checkIfExists: true) def ch_methods_description = channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: true)) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{_meta, file -> file}) ch_multiqc_files = ch_multiqc_files.mix(CREATE_MANIFESTS.out.upload_registered_mags) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{_meta, file -> file}) MULTIQC( ch_multiqc_files.flatten().collect().map { files -> [ diff --git a/workflows/readsubmit.nf b/workflows/readsubmit.nf index 547b4667..8bdff882 100644 --- a/workflows/readsubmit.nf +++ b/workflows/readsubmit.nf @@ -43,38 +43,20 @@ workflow READSUBMIT { def ch_versions = channel.empty() def ch_multiqc_files = channel.empty() - // Create reads channel with proper metadata structure + // --------- Create reads channel with proper metadata structure reads_ch = ch_samplesheet - .map { row -> - def (meta_in, sample_accession, fastq_1, fastq_2, - platform, instrument, library_source, library_selection, library_strategy, - insert_size, library_name, description) = row - def meta = [ - id: meta_in.id, - sample_accession: sample_accession, - single_end: fastq_2 ? false : true, - platform: platform, - instrument: instrument, - library_source: library_source, - library_selection: library_selection, - library_strategy: library_strategy, - insert_size: insert_size ?: null, - library_name: library_name ?: null, - description: description ?: null - ] - - if (fastq_2 && fastq_2 != "") { + .map { meta, reads_1, reads_2 -> + def new_meta = meta + [single_end: !reads_2] + if ( !new_meta.single_end ) { // If paired end reads - [meta, [file(fastq_1), file(fastq_2)]] + [new_meta, [reads_1, reads_2]] } else { // If single end - [meta, file(fastq_1)] + [new_meta, reads_1] } } - if (!submission_study && !study_metadata) { - error("Either --submission_study or --study_metadata must be provided") - } + // --------- Register study if accession was not provided via --submission_study def study_accession_ch if (submission_study) { // Use provided study accession directly @@ -94,7 +76,7 @@ workflow READSUBMIT { } } - // Generate reads manifest files + // --------- Generate reads manifest files CREATE_READS_MANIFEST( reads_ch, study_accession_ch.first() @@ -106,6 +88,7 @@ workflow READSUBMIT { [meta, fastq, manifest] } + // --------- Upload data to ENA SUBMIT ( submission_input, test_upload, @@ -119,9 +102,7 @@ workflow READSUBMIT { true // skip_header - we want to keep the header from the first file and skip it for the rest ) - // - // Collate and save software versions - // + // --------- Collate and save software versions def topic_versions = channel.topic("versions") .distinct() .branch { entry -> @@ -148,9 +129,7 @@ workflow READSUBMIT { newLine: true ) - // - // MODULE: MultiQC - // + // --------- MODULE: MultiQC ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) def ch_summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") def ch_workflow_summary = channel.value(paramsSummaryMultiqc(ch_summary_params))