From 8c0954e1c9023724e265cb6b035f93265cb8dbb4 Mon Sep 17 00:00:00 2001 From: vagkaratzas Date: Sat, 26 Sep 2026 10:33:25 +0100 Subject: [PATCH 1/3] Fix InterProScan ignoring staged database (#114) Update the interproscan module to pick up nf-core/modules#13009, so the staged database is actually used. Default interproscan_db_url to the 5.59-91.0 release matching the container, pass the inner data/ folder of an untarred release to INTERPROSCAN, and drop Hamap from test profiles (test tarball ships hamap/2023_05, 5.59 expects 2021_04). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++ conf/test.config | 2 +- conf/test_full.config | 2 +- docs/usage.md | 57 ++++++++++--------- modules.json | 2 +- modules/nf-core/interproscan/main.nf | 8 ++- modules/nf-core/interproscan/meta.yml | 8 ++- .../tests/nextflow_database.config | 2 +- nextflow.config | 2 +- nextflow_schema.json | 6 +- .../local/functional_annotation/main.nf | 4 ++ .../tests/nextflow.config | 2 +- 12 files changed, 58 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1633f..1a6d954 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#111](https://github.com/nf-core/proteinannotator/pull/111) - nf-core tools template update to 4.1.0. (by @vagkaratzas) - [#106](https://github.com/nf-core/proteinannotator/pull/106) - nf-core tools template update to 4.0.3. (by @vagkaratzas) +### `Fixed` + +- [#114](https://github.com/nf-core/proteinannotator/issues/114) - Fixed `INTERPROSCAN` failing on real databases: updated the `interproscan` module so it uses the staged database ([nf-core/modules#13009](https://github.com/nf-core/modules/issues/13009)), set the default `interproscan_db_url` to the 5.59-91.0 release that matches the container, and pass the inner `data/` directory of an untarred release to `INTERPROSCAN`. (by @vagkaratzas) + ### `Dependencies` | Tool | Previous version | New version | diff --git a/conf/test.config b/conf/test.config index 5213e9d..0bbb989 100644 --- a/conf/test.config +++ b/conf/test.config @@ -31,7 +31,7 @@ params { metagroot_latest_link = params.pipelines_testdata_base_path + 'proteinannotator/testdata/metagroot/metagroot_test.hmm.gz' // Functional annotation interproscan_db_url = params.pipelines_testdata_base_path + 'proteinannotator/testdata/interproscan/interproscan_test.tar.gz' - interproscan_applications = 'Hamap,TIGRFAM,sfld' + interproscan_applications = 'TIGRFAM,sfld' kofamscan_profiles_url = params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kofamscan/profiles.tar.gz' kofamscan_ko_list_url = params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kofamscan/ko_list.gz' } diff --git a/conf/test_full.config b/conf/test_full.config index fc457ec..036608d 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -23,7 +23,7 @@ params { metagroot_latest_link = params.pipelines_testdata_base_path + 'proteinannotator/testdata/metagroot/metagroot_test.hmm.gz' // Functional annotation interproscan_db_url = params.pipelines_testdata_base_path + 'proteinannotator/testdata/interproscan/interproscan_test.tar.gz' - interproscan_applications = 'Hamap,TIGRFAM,sfld' + interproscan_applications = 'TIGRFAM,sfld' kofamscan_profiles_url = params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kofamscan/profiles.tar.gz' kofamscan_ko_list_url = params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kofamscan/ko_list.gz' } diff --git a/docs/usage.md b/docs/usage.md index 83cb471..f31b812 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,43 +85,44 @@ You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-c [InterProScan](https://github.com/ebi-pf-team/interproscan) is used to provide more information about the proteins annotated on the contigs. By default, turning on this subworkflow without `--skip_interproscan` will download and unzip the InterPro database. The database will then be saved in the output directory `/downloaded_dbs/interproscan_db/`. We recommend keeping a copy of this directory for future reuse in case the results folder is deleted. > [!NOTE] -> The large database download (5.5GB) can take up to 4 hours depending on the bandwidth. +> The large database download (6.6GB) can take up to 4 hours depending on the bandwidth. -A local version of the database can be supplied to the pipeline by passing the InterProScan database directory to `--interproscan_db `. The directory can be created by running (e.g. for database version 5.72-103.0): +The database release must match the InterProScan version in the pipeline's container, which is **5.59-91.0**. Newer releases rename or replace member databases (e.g. TIGRFAM was merged into NCBIfam) and will not be found. -``` -curl -L https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.72-103.0/interproscan-5.72-103.0-64-bit.tar.gz -o interproscan_db/interproscan-5.72-103.0-64-bit.tar.gz -tar -xzf interproscan_db/interproscan-5.72-103.0-64-bit.tar.gz -C interproscan_db/ +A local version of the database can be supplied to the pipeline with `--interproscan_db`. This can be either the untarred release directory or the `data/` directory inside it. The directory can be created by running: +```bash +curl -L https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.59-91.0/interproscan-5.59-91.0-64-bit.tar.gz -o interproscan-5.59-91.0-64-bit.tar.gz +tar -pxzf interproscan-5.59-91.0-64-bit.tar.gz ``` -The contents of the database directory should include the directory `data` in the top level with a number of subdirectories: +This creates the following layout: ``` -interproscan_db/ - └── data/ - ├── antifam - ├── cdd - ├── funfam - ├── gene3d - ├── hamap - ├── ncbifam - ├── panther - | └── [18.0] - ├── pfam - | └── [36.0] - ├── phobius - ├── pirsf - ├── pirsr - ├── prints - ├── prosite - | └── [2023_05] - ├── sfld - ├── smart - ├── superfamily - └── tmhmm +interproscan-5.59-91.0/ <- pass this (or its data/ subdirectory) to --interproscan_db + ├── bin/ + ├── data/ + │ ├── hamap/ + │ ├── panther/ + │ ├── pirsf/ + │ ├── sfld/ + │ ├── tigrfam/ + │ └── ... + ├── interproscan.properties + ├── interproscan.sh + └── setup.py ``` +> [!NOTE] +> The HMMs in the 5.59-91.0 release are not pre-indexed. The default applications do not need them indexed, except PIRSF, which indexes its own HMMs the first time it runs. When several samples run InterProScan in parallel against a fresh database, they may all try to index the same PIRSF files at once. To avoid this, index the database once before running the pipeline, from inside the untarred release directory (the one containing `setup.py` and `interproscan.properties`, not `data/`): +> +> ```bash +> cd interproscan-5.59-91.0/ +> python3 setup.py -f interproscan.properties +> ``` +> +> For a database downloaded by the pipeline, run the same command inside `/downloaded_dbs/interproscan_db/`, then reuse that directory with `--interproscan_db`. + ### KOfamScan [KOfamScan](https://github.com/takaram/kofam_scan) assigns KEGG Orthologs to protein sequences using the KOfam profile HMM database and KO-specific score thresholds. It runs by default and can be disabled with `--skip_kofamscan`. diff --git a/modules.json b/modules.json index 4bcf981..bb0c3df 100644 --- a/modules.json +++ b/modules.json @@ -23,7 +23,7 @@ }, "interproscan": { "branch": "master", - "git_sha": "6d46786420b4d7bc88eba026eb389c0c5535d120", + "git_sha": "3137cddd6fa5fd47e1673081fa82a12644e91996", "installed_by": ["modules"] }, "kofamscan": { diff --git a/modules/nf-core/interproscan/main.nf b/modules/nf-core/interproscan/main.nf index 0afeb91..67e75e9 100644 --- a/modules/nf-core/interproscan/main.nf +++ b/modules/nf-core/interproscan/main.nf @@ -37,8 +37,12 @@ process INTERPROSCAN { INTERPROSCAN_DIR="\$( dirname "\$( dirname "\$( which interproscan.sh )" )" )" INTERPROSCAN_PROPERTIES="\$( find "\$INTERPROSCAN_DIR/share" -name "interproscan.properties" )" cp "\$INTERPROSCAN_PROPERTIES" . - sed -i "/^bin\\.directory=/ s|.*|bin.directory=\$INTERPROSCAN_DIR/bin|" interproscan.properties - export INTERPROSCAN_CONF=interproscan.properties + sed -i \\ + -e "s|^bin\\.directory=.*|bin.directory=\$( dirname "\$INTERPROSCAN_PROPERTIES" )/bin|" \\ + -e "s|^data\\.directory=.*|data.directory=\$PWD/data|" \\ + interproscan.properties + # must be absolute: interproscan.sh cd's into its install dir before reading it + export INTERPROSCAN_CONF=\$PWD/interproscan.properties fi # else use sample DB included with conda ( testing only! ) interproscan.sh \\ diff --git a/modules/nf-core/interproscan/meta.yml b/modules/nf-core/interproscan/meta.yml index cd58f2b..05c986c 100644 --- a/modules/nf-core/interproscan/meta.yml +++ b/modules/nf-core/interproscan/meta.yml @@ -31,8 +31,12 @@ input: - edam: http://edamontology.org/format_1929 # FASTA - interproscan_database: type: directory - description: Path to the interproscan database (untarred - http://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/${version_major}-${version_minor}/interproscan-${version_major}-${version_minor}-64-bit.tar.gz) + description: | + Path to the data/ folder of an InterProScan release matching the module + version (untarred from + http://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/${version_major}-${version_minor}/interproscan-${version_major}-${version_minor}-64-bit.tar.gz). + HMMs should already be pressed (python3 setup.py -f interproscan.properties), + otherwise hmmpress runs in every task and parallel tasks may race. output: tsv: - - meta: diff --git a/modules/nf-core/interproscan/tests/nextflow_database.config b/modules/nf-core/interproscan/tests/nextflow_database.config index bc8c773..0a9590e 100644 --- a/modules/nf-core/interproscan/tests/nextflow_database.config +++ b/modules/nf-core/interproscan/tests/nextflow_database.config @@ -1,6 +1,6 @@ process { withName: INTERPROSCAN { - ext.args = '--applications Hamap,TIGRFAM,sfld --disable-precalc' + ext.args = '--applications TIGRFAM,sfld --disable-precalc' memory = 12.GB cpus = 1 } diff --git a/nextflow.config b/nextflow.config index e0a8b01..409f640 100644 --- a/nextflow.config +++ b/nextflow.config @@ -35,7 +35,7 @@ params { // Functional annotation skip_interproscan = false - interproscan_db_url = "https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.72-103.0/interproscan-5.72-103.0-64-bit.tar.gz" + interproscan_db_url = "https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.59-91.0/interproscan-5.59-91.0-64-bit.tar.gz" interproscan_db = null interproscan_applications = 'Hamap,PANTHER,PIRSF,TIGRFAM,sfld' interproscan_enableprecalc = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 6078fd6..ddeb8f7 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -324,14 +324,14 @@ }, "interproscan_db_url": { "type": "string", - "default": "https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.72-103.0/interproscan-5.72-103.0-64-bit.tar.gz", - "help_text": "This allows the user to change the InterProScan database version that the pipeline will download for you automatically. To instead use a pre-downloaded database, please supply its path to `--interproscan_db`. Changing this URL allows for the use of the latest database release. By default this is set to `http://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.72-103.0/interproscan-5.72-103.0-64-bit.tar.gz`. ", + "default": "https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.59-91.0/interproscan-5.59-91.0-64-bit.tar.gz", + "help_text": "This allows the user to change the InterProScan database version that the pipeline will download for you automatically. To instead use a pre-downloaded database, please supply its path to `--interproscan_db`. The release must match the InterProScan version of the pipeline's container (5.59-91.0), so do not point this at a newer release. By default this is set to `https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.59-91.0/interproscan-5.59-91.0-64-bit.tar.gz`.", "description": "Change the database version used for annotation.", "fa_icon": "fas fa-database" }, "interproscan_db": { "type": "string", - "help_text": "Use this to supply the path to a pre-downloaded InterProScan database. This can be any unzipped InterProScan version.\n\nFor more details on where to find different InterProScan databases see tool [documentation](https://interproscan-docs.readthedocs.io/en/latest/UserDocs.html#obtaining-a-copy-of-interproscan).\n", + "help_text": "Use this to supply the path to a pre-downloaded InterProScan database. This must be an untarred InterProScan 5.59-91.0 release, matching the pipeline's container. Either the release's top-level directory or its inner `data/` directory can be supplied.\n\nFor more details on where to find different InterProScan databases see tool [documentation](https://interproscan-docs.readthedocs.io/en/latest/UserDocs.html#obtaining-a-copy-of-interproscan).\n", "description": "Path to pre-downloaded InterProScan database.", "fa_icon": "fas fa-database" }, diff --git a/subworkflows/local/functional_annotation/main.nf b/subworkflows/local/functional_annotation/main.nf index f14f948..9ebbd9f 100644 --- a/subworkflows/local/functional_annotation/main.nf +++ b/subworkflows/local/functional_annotation/main.nf @@ -36,6 +36,10 @@ workflow FUNCTIONAL_ANNOTATION { .first() } + // A full release untars to the whole install; INTERPROSCAN needs its inner data/ folder + ch_interproscan_db = ch_interproscan_db + .map { database -> database.resolve('data').isDirectory() ? database.resolve('data') : database } + INTERPROSCAN( ch_fasta, ch_interproscan_db ) ch_interproscan_tsv = INTERPROSCAN.out.tsv } diff --git a/subworkflows/local/functional_annotation/tests/nextflow.config b/subworkflows/local/functional_annotation/tests/nextflow.config index f55bf66..22fd313 100644 --- a/subworkflows/local/functional_annotation/tests/nextflow.config +++ b/subworkflows/local/functional_annotation/tests/nextflow.config @@ -1,6 +1,6 @@ process { withName: INTERPROSCAN { - ext.args = '--applications Hamap,TIGRFAM,sfld --disable-precalc' + ext.args = '--applications TIGRFAM,sfld --disable-precalc' memory = 12.GB cpus = 1 } From cbf1cf4ff2461cd28933ccc425779af3975a38b6 Mon Sep 17 00:00:00 2001 From: vagkaratzas Date: Sat, 26 Sep 2026 10:33:45 +0100 Subject: [PATCH 2/3] Update CHANGELOG with PR number Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a6d954..61d7989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#114](https://github.com/nf-core/proteinannotator/issues/114) - Fixed `INTERPROSCAN` failing on real databases: updated the `interproscan` module so it uses the staged database ([nf-core/modules#13009](https://github.com/nf-core/modules/issues/13009)), set the default `interproscan_db_url` to the 5.59-91.0 release that matches the container, and pass the inner `data/` directory of an untarred release to `INTERPROSCAN`. (by @vagkaratzas) +- [#115](https://github.com/nf-core/proteinannotator/pull/115) - Fixed `INTERPROSCAN` failing on real databases: updated the `interproscan` module so it uses the staged database ([nf-core/modules#13009](https://github.com/nf-core/modules/issues/13009)), set the default `interproscan_db_url` to the 5.59-91.0 release that matches the container, and pass the inner `data/` directory of an untarred release to `INTERPROSCAN`. (by @vagkaratzas) ### `Dependencies` From aa7039267b94e57610fdcdf6cffd86cbaecdf7ba Mon Sep 17 00:00:00 2001 From: vagkaratzas Date: Sun, 27 Sep 2026 08:34:40 +0100 Subject: [PATCH 3/3] Drop PIRSF from default InterProScan applications On a fresh pipeline-downloaded database, parallel INTERPROSCAN tasks race to hmmpress the same PIRSF HMMs. Keep it opt-in; usage docs explain how to pre-index and re-enable it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- docs/output.md | 2 +- docs/usage.md | 2 +- nextflow.config | 2 +- nextflow_schema.json | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d7989..7ae7c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#115](https://github.com/nf-core/proteinannotator/pull/115) - Fixed `INTERPROSCAN` failing on real databases: updated the `interproscan` module so it uses the staged database ([nf-core/modules#13009](https://github.com/nf-core/modules/issues/13009)), set the default `interproscan_db_url` to the 5.59-91.0 release that matches the container, and pass the inner `data/` directory of an untarred release to `INTERPROSCAN`. (by @vagkaratzas) +- [#115](https://github.com/nf-core/proteinannotator/pull/115) - Fixed `INTERPROSCAN` failing on real databases: updated the `interproscan` module so it uses the staged database ([nf-core/modules#13009](https://github.com/nf-core/modules/issues/13009)), set the default `interproscan_db_url` to the 5.59-91.0 release that matches the container, pass the inner `data/` directory of an untarred release to `INTERPROSCAN`, and removed `PIRSF` from the default `interproscan_applications`, since parallel samples can race to index its HMMs on a fresh database (see usage docs to re-enable it). (by @vagkaratzas) ### `Dependencies` diff --git a/docs/output.md b/docs/output.md index 7caea22..ec26322 100644 --- a/docs/output.md +++ b/docs/output.md @@ -128,7 +128,7 @@ Each of the `domain_annotation/` subfolders (e.g., `pfam`, `funfam`, `nmpfams`, [InterProScan](https://interproscan-docs.readthedocs.io/en/v5/#) is a protein annotation tool that searches [InterPro](http://www.ebi.ac.uk/interpro/), a database which integrates predictive information about protein function from a number of member resources, giving an overview of the families that a protein belongs to and the domains and sites it contains. The default database applications that are used to functionally annotate sequences include -Hamap, PANTHER, PIRSF, TIGRFAM and sfld, and are set through the `--interproscan_applications` parameter. +Hamap, PANTHER, TIGRFAM and sfld, and are set through the `--interproscan_applications` parameter. See also [InterProScan output documentation](https://interproscan-docs.readthedocs.io/en/v5/), where most of these examples are taken from. diff --git a/docs/usage.md b/docs/usage.md index f31b812..4189928 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -114,7 +114,7 @@ interproscan-5.59-91.0/ <- pass this (or its data/ subdirectory) to --i ``` > [!NOTE] -> The HMMs in the 5.59-91.0 release are not pre-indexed. The default applications do not need them indexed, except PIRSF, which indexes its own HMMs the first time it runs. When several samples run InterProScan in parallel against a fresh database, they may all try to index the same PIRSF files at once. To avoid this, index the database once before running the pipeline, from inside the untarred release directory (the one containing `setup.py` and `interproscan.properties`, not `data/`): +> The HMMs in the 5.59-91.0 release are not pre-indexed. The default applications do not need them indexed. PIRSF is not in the defaults because it indexes its own HMMs the first time it runs, and when several samples run InterProScan in parallel against a fresh database, they may all try to index the same PIRSF files at once. To add PIRSF to `--interproscan_applications`, index the database once before running the pipeline, from inside the untarred release directory (the one containing `setup.py` and `interproscan.properties`, not `data/`): > > ```bash > cd interproscan-5.59-91.0/ diff --git a/nextflow.config b/nextflow.config index 409f640..fe94e08 100644 --- a/nextflow.config +++ b/nextflow.config @@ -37,7 +37,7 @@ params { skip_interproscan = false interproscan_db_url = "https://ftp.ebi.ac.uk/pub/software/unix/iprscan/5/5.59-91.0/interproscan-5.59-91.0-64-bit.tar.gz" interproscan_db = null - interproscan_applications = 'Hamap,PANTHER,PIRSF,TIGRFAM,sfld' + interproscan_applications = 'Hamap,PANTHER,TIGRFAM,sfld' interproscan_enableprecalc = false skip_kofamscan = false kofamscan_profiles_url = "https://www.genome.jp/ftp/db/kofam/profiles.tar.gz" diff --git a/nextflow_schema.json b/nextflow_schema.json index ddeb8f7..2f0d698 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -337,9 +337,9 @@ }, "interproscan_applications": { "type": "string", - "default": "Hamap,PANTHER,PIRSF,TIGRFAM,sfld", + "default": "Hamap,PANTHER,TIGRFAM,sfld", "pattern": "^\\w+(,\\w+)*", - "help_text": "A comma-separated string specifying the database(s) to be used to annotate the coding regions annotated during the contig annotation workflow of the pipeline. By default these include `Hamap,PANTHER,PIRSF,TIGRFAM,sfld`.\n\nFor more information about all possible databases see the tool [documentation](https://interproscan-docs.readthedocs.io/en/v5/HowToRun.html).\n\n> Modifies tool parameter(s):\n> - InterProScan: `--applications`", + "help_text": "A comma-separated string specifying the database(s) to be used to annotate the coding regions annotated during the contig annotation workflow of the pipeline. By default these include `Hamap,PANTHER,TIGRFAM,sfld`.\n\nFor more information about all possible databases see the tool [documentation](https://interproscan-docs.readthedocs.io/en/v5/HowToRun.html).\n\n> Modifies tool parameter(s):\n> - InterProScan: `--applications`", "description": "Assigns the database(s) to be used to annotate the coding regions.", "fa_icon": "fas fa-database" },