diff --git a/.Rbuildignore b/.Rbuildignore index 87b9f63c..a373fd95 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -38,9 +38,9 @@ ^refCheck\.R$ ^\.Rproj\.user$ ^local_validation$ -^R/hypeR_examples\.R$ ^testing_manuscripts$ ^artifacts$ ^sigs_in_repo\.csv$ ^\.lintr$ ^CONTRIBUTING\.md$ +^\.superpowers$ diff --git a/.claude/skills/connect-sigrepo-mcp/SKILL.md b/.claude/skills/connect-sigrepo-mcp/SKILL.md index f7ff92ec..9d8594be 100644 --- a/.claude/skills/connect-sigrepo-mcp/SKILL.md +++ b/.claude/skills/connect-sigrepo-mcp/SKILL.md @@ -1,6 +1,6 @@ --- name: connect-sigrepo-mcp -description: Registers SigRepo's MCP server (https://sigrepo.org/mcp/) with Claude Code if it isn't already added, asks the user for their SigRepo api_key, and carries it forward for the rest of the session so every MCP tool call (search_signatures, run_enrichment, etc.) includes it automatically. Use this whenever the user asks to "connect to SigRepo's MCP," "set up SigRepo MCP," "use the SigRepo MCP tools," or wants to start querying/searching SigRepo signatures, collections, gene sets, or run enrichment via MCP but hasn't already supplied an api_key this session. +description: Registers SigRepo's MCP server (https://sigrepo.org/mcp/) with Claude Code if it isn't already added, asks the user for their SigRepo api_key, and carries it forward for the rest of the session so every MCP tool call (search_signatures, compare_signatures, etc.) includes it automatically. Use this whenever the user asks to "connect to SigRepo's MCP," "set up SigRepo MCP," "use the SigRepo MCP tools," or wants to start querying/searching SigRepo signatures, collections, gene sets, or run enrichment via MCP but hasn't already supplied an api_key this session. --- # Connect to SigRepo's MCP server @@ -53,7 +53,7 @@ it to a file, and don't echo it back after this point. The registered server exposes 9 tools: `list_vocabulary`, `search_signatures`, `get_signature_context`, `compare_signatures`, `search_collections`, `search_geneset_resources`, `search_geneset_entries`, -`search_features`, and `run_enrichment`. A reasonable starting point if the +`search_features`, and `rummagene_enrich`. A reasonable starting point if the person hasn't said what they want yet: call `list_vocabulary` to show what organisms/phenotypes/assay types actually have data, then `search_signatures` from there. diff --git a/.github/workflows/R-CMD-CHECK.yaml b/.github/workflows/R-CMD-CHECK.yaml index 239d9beb..25d6cdfd 100755 --- a/.github/workflows/R-CMD-CHECK.yaml +++ b/.github/workflows/R-CMD-CHECK.yaml @@ -2,7 +2,7 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [master] + branches: [master, dev] paths-ignore: - '.github/workflows/document.yaml' - '.github/workflows/gh-pages-pkgdown.yaml' @@ -12,7 +12,7 @@ on: - '**.Rmd' - '.gitignore' pull_request: - branches: [master] + branches: [master, dev] release: types: [published] workflow_dispatch: @@ -43,6 +43,13 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes + # test_conn_handler defaults host to sigrepo.org and api_host to the + # production IP. Nothing here reaches production today: this check supplies + # no SIGREPO_TEST_DB_USER or _PASSWORD, runs with --no-tests --no-examples, + # and every @examples block is wrapped in \dontrun. Pin the hosts anyway, so + # that stays true if credentials are ever added to this workflow. + SIGREPO_TEST_DB_HOST: 127.0.0.1 + SIGREPO_TEST_API_HOST: http://127.0.0.1 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/close-linked-issues.yml b/.github/workflows/close-linked-issues.yml new file mode 100644 index 00000000..bebc3127 --- /dev/null +++ b/.github/workflows/close-linked-issues.yml @@ -0,0 +1,72 @@ +name: close-linked-issues + +on: + # pull_request_target, not pull_request: the job needs a token that can close + # issues, and it never checks out or runs the pull request's code, so the + # usual risk of that trigger does not apply. + pull_request_target: + types: [closed] + # GitHub honours `Closes #NNN` only when a pull request merges into the + # default branch, which is master. Pull requests target dev, so without + # this job the keyword does nothing until a release carries dev into + # master, and every issue resolved in between sits open on the board. + branches: [dev] + +permissions: + issues: write + +jobs: + close: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v7 + with: + script: | + const pr = context.payload.pull_request; + const { owner, repo } = context.repo; + + // Keywords inside HTML comments and code are not instructions. + const body = (pr.body || '') + .replace(//g, '') + .replace(/```[\s\S]*?```/g, '') + .replace(/`[^`\n]*`/g, ''); + + // The same keywords GitHub recognises, followed by #N, owner/repo#N + // or an issue URL. + const pattern = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:([\w.-]+\/[\w.-]+)?#(\d+)|https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/issues\/(\d+))/gi; + + const numbers = new Set(); + for (const m of body.matchAll(pattern)) { + const target = (m[1] || m[3] || `${owner}/${repo}`).toLowerCase(); + const number = Number(m[2] || m[4]); + if (target !== `${owner}/${repo}`.toLowerCase()) { + // This job's token reaches this repository only. + core.warning(`${target}#${number} is in another repository; close it by hand.`); + continue; + } + numbers.add(number); + } + + if (numbers.size === 0) { + core.info(`#${pr.number} names no issue to close.`); + return; + } + + for (const issue_number of numbers) { + const { data: issue } = await github.rest.issues.get({ owner, repo, issue_number }); + if (issue.pull_request || issue.state === 'closed') { + core.info(`#${issue_number} skipped: ${issue.pull_request ? 'a pull request' : 'already closed'}.`); + continue; + } + await github.rest.issues.createComment({ + owner, repo, issue_number, + body: `Closed by #${pr.number}, merged into \`${pr.base.ref}\`.`, + }); + await github.rest.issues.update({ + owner, repo, issue_number, + state: 'closed', + state_reason: 'completed', + }); + core.info(`#${issue_number} closed.`); + } diff --git a/.github/workflows/notify-staging-image.yml b/.github/workflows/notify-staging-image.yml new file mode 100644 index 00000000..a6683cab --- /dev/null +++ b/.github/workflows/notify-staging-image.yml @@ -0,0 +1,77 @@ +name: notify staging image + +# Tell SigRepo_Server to rebuild the staging image when this package changes on +# dev. +# +# Why this is needed at all: the server calls SigRepo:: by namespace, which +# resolves from the installed library, not from the /SigRepo bind mount -- +# inside the container find.package("SigRepo") returns +# /usr/local/lib/R/site-library/SigRepo, and nothing calls pkgload::load_all() +# on that path. So this package reaches the server ONLY through the Docker +# image. SigRepo_Server's build watches only its own files, so without this +# workflow a push here triggers nothing and montilab.bu.edu quietly keeps +# running whatever client was baked in last. +# +# That gap matters most for R/sqlFunctions.R: lookup_table_sql is called 21 +# times across 5 of the server's API files and twice in its Shiny app, and its +# query shape is what made getSignature take 4.6 seconds before the index fix. +# Only the staging replica has enough rows for a regression to show. + +on: + push: + branches: [dev] + # Only what changes the INSTALLED package. A vignette or README push does + # not need a nine minute image build. + paths: + - "R/**" + - "src/**" + - "inst/**" + - "DESCRIPTION" + - "NAMESPACE" + - ".github/workflows/notify-staging-image.yml" + workflow_dispatch: + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Ask SigRepo_Server to rebuild montilab/sigrepo:dev + env: + # secrets.GITHUB_TOKEN cannot reach another repository. GH_DOCS is the + # cross-repo token this project already uses; it needs access to + # montilab/SigRepo_Server for this to work. + TOKEN: ${{ secrets.GH_DOCS }} + SHA: ${{ github.sha }} + REF: ${{ github.ref_name }} + run: | + set -uo pipefail + + if [ -z "${TOKEN:-}" ]; then + echo "::error::GH_DOCS is not set, so the staging image cannot be rebuilt." + echo "::error::Staging would silently keep running an older client." + exit 1 + fi + + code=$(curl -s -o /tmp/dispatch.out -w '%{http_code}' -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/montilab/SigRepo_Server/dispatches \ + -d "{\"event_type\":\"sigrepo-client-updated\",\"client_payload\":{\"ref\":\"$REF\",\"sha\":\"$SHA\"}}") + + if [ "$code" = "204" ]; then + echo "Asked SigRepo_Server to rebuild montilab/sigrepo:dev for $REF@${SHA:0:8}" + exit 0 + fi + + # 404 rather than 403 is the usual failure here: GitHub will not + # confirm the existence of a repository the token cannot see, so an + # under-scoped token looks exactly like a missing one. Say so, because + # failing quietly means staging goes stale while looking healthy. + echo "::error::Dispatch to montilab/SigRepo_Server failed with HTTP $code." + if [ "$code" = "404" ]; then + echo "::error::404 here usually means GH_DOCS cannot see SigRepo_Server," + echo "::error::not that the repository is missing. Check the token's scope." + fi + cat /tmp/dispatch.out + exit 1 diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 9b2f0500..861e2e73 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -28,6 +28,14 @@ jobs: # the site shows the example code without output. SIGREPO_TEST_DB_USER: ${{ secrets.SIGREPO_TEST_DB_USER }} SIGREPO_TEST_DB_PASSWORD: ${{ secrets.SIGREPO_TEST_DB_PASSWORD }} + # These two are the reason this matters. test_conn_handler still DEFAULTS + # host to sigrepo.org and api_host to the production IP, and the vignettes + # gate evaluation on the credentials alone, not on the host. So the moment + # the secrets above are set, an unpinned build would run addSignature() + # and deleteSignature() against PRODUCTION, which is read-only for + # signatures by policy. Point them at a stack that is safe to write to. + SIGREPO_TEST_DB_HOST: 127.0.0.1 + SIGREPO_TEST_API_HOST: http://127.0.0.1 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 53b0b55e..447b80e1 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -2,9 +2,9 @@ name: validation on: pull_request: - branches: [master] + branches: [master, dev] push: - branches: [master] + branches: [master, dev] paths: - "R/**" - "tests/**" @@ -133,8 +133,10 @@ jobs: # # /init_db is schema + reference tables + a full MSigDB gene set build, # and msigdbr downloads that data from zenodo.org on first use. No test in - # this package uses MSigDB: the hypeR and hypeR-GEM tests construct their - # own gene sets inline and pass them to runHypeR(). So the download bought + # this package needs the database's MSigDB gene sets: the hypeR and + # hypeR-GEM client tests construct their own gene sets inline, and the one + # MSigDB test, test_hypeR_genesets.R, does a live Hallmark fetch through + # msigdbr on the client side that skips on error. So the download bought # this suite nothing and made every run depend on a third-party service # staying up -- which is how a zenodo 504 turned into a red build here. - name: Initialize schema and reference data diff --git a/.gitignore b/.gitignore index 53e124ac..b2dfd4b0 100755 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ # Ignore folders .Rproj.user/ -docs/ +docs/* # Local-only validation harness: runs against a developer's own MySQL/API # stack via local_validation/.env.local-validation. Never belongs on a remote. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53b5eeac..4c08de9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,16 +15,40 @@ are linked before any code is written: ```sh gh issue create --repo montilab/SigRepo --title "..." --body "..." -gh issue develop --base master --checkout +gh issue develop --base dev --checkout ``` Branch names carry no type prefix — not `fix/`, not `feat/`, not `chore/`. The default `-` that `gh` produces is what we want. +**`dev` is what gets worked on; `master` is what is live.** Branches come off +`dev` and pull requests go into `dev`. `dev` is deployed to the staging instance +at montilab.bu.edu, where a change is exercised against a full replica of the +data before anyone outside sees it. + +`master` is production: it is what sigrepo.org runs. It gains commits only +through a release pull request from `dev`, and direct pushes to it are blocked. +If something ever has to be hotfixed straight onto `master`, merge `master` back +into `dev` the same day. That one rule is what stops the two branches drifting +apart, which is how this arrangement usually goes wrong. + +A change that spans both repositories is released from both or from neither. +Otherwise production runs a client and a server from different lines, and the +resulting bug report makes no sense to anyone. + **Every pull request closes its issue.** Put `Closes #NNN` in the description. This is the rule the rest depends on: it is what lets the board move by itself and what leaves the reason for a change next to the change. +Note what closes it. GitHub itself honours `Closes #NNN` only when a pull request +merges into the repository's default branch, which is `master`, and pull requests +target `dev`. The `close-linked-issues` workflow covers the gap: when a pull +request merges into `dev`, it closes each issue the description names and leaves +a comment naming the pull request. So a closed issue means merged into `dev`, not +live; what is live is whatever the last release carried into `master`. The +workflow reaches its own repository only, so an issue in the other repository +that your pull request resolves has to be closed by hand. + **Ideas and work are different things.** An idea states a goal, is labelled `idea`, and is never closed by a pull request. A task or bug is real work: it links to the idea it serves, and it is closed by the pull request that resolves it. An idea is @@ -56,6 +80,30 @@ in the issue, or label it `needs-decision` and say what the decision is. nobody answers. Closing is not a judgement on the idea; reopen it when it matters again. +## Reaching the staging instance + +montilab.bu.edu keeps every port bound to `127.0.0.1`. It is a shared, +BU-managed host, the lab has already been warned once about an exposed port, and +the staging database holds real signatures behind accounts that can edit them. +So nothing there is published to the network, and reaching it means your own SSH +tunnel with your own BU account: + +```sh +ssh -N -o ExitOnForwardFailure=yes \ + -L 9051:127.0.0.1:8051 -L 9050:127.0.0.1:8050 -L 9020:127.0.0.1:8020 montilab +``` + +with `Host montilab` proxying through `scc4.bu.edu` — not scc2, which has no +route to the BUMC network. Local ports are offset by 1000 so they do not collide +with a stack running on your own machine. If you already have a ControlMaster +session open, that command returns immediately and the master process holds the +forwards: the tunnel is live even though it looks like the command exited. + +Then open . + +This needs SCC access. If you do not have it, ask for a walkthrough rather than +a port. + ## Labels Exactly one type label, and exactly one status label. diff --git a/DESCRIPTION b/DESCRIPTION index 999f0980..ffa94046 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: SigRepo Type: Package Title: Storing and processing omic signatures and collections -Version: 1.0.0 +Version: 1.1.0 Authors@R: c(person(given="Reina", family="Chau", role=c("aut","cre"), email="rchau88@bu.edu", comment=c(ORCID="0000-0003-3012-1404")), @@ -23,10 +23,16 @@ Imports: DBI, digest, dplyr, + fgsea, + ggplot2, httr, + hypeR (>= 2.0.0), + hypeR.GEM, jsonlite, methods, + msigdbr, OmicSignature (>= 1.4.1), + openxlsx, purrr, readr, rlang, @@ -37,9 +43,6 @@ Remotes: Suggests: BiocManager, devtools, - fgsea, - hypeR, - hypeR.GEM, knitr, pkgdown, rmarkdown, diff --git a/NAMESPACE b/NAMESPACE index 752de258..aef0c679 100755 --- a/NAMESPACE +++ b/NAMESPACE @@ -50,17 +50,27 @@ export(delete_table_sql) export(getAPIKey) export(getCollection) export(getDBColNames) +export(getHypeRDifexp) +export(getHypeRGenesets) export(getNumOfObs) export(getSignature) export(getSignatureFeatureSet) export(getVariableID) export(get_api_handle) export(get_handle) +export(hypeRDotData) +export(hypeREnrichmentData) +export(hypeRSignatureKey) +export(hypeRToExcel) export(inActivateUser) export(init_handle) export(insert_table_sql) export(lookup_table_sql) +export(mergeDifexp) export(newConnHandler) +export(plotHypeRDots) +export(plotHypeREnrichment) +export(plotHypeRMap) export(prepareHypeRSignatures) export(print_messages) export(protTransform) @@ -70,6 +80,7 @@ export(runHypeR) export(runHyperGEM) export(searchAssayType) export(searchCollection) +export(searchExternal) export(searchGenesetEntry) export(searchGenesetResource) export(searchGeneticVariantsFeatureSet) @@ -102,9 +113,11 @@ export(updateTranscriptomicsFeatureSet) export(updateUser) export(user_grant_statements) export(verbose) +export(writeSignatureSearchScript) import(DBI) import(OmicSignature) import(RMySQL) import(biomaRt) import(digest) import(readr) +importFrom(rlang,.data) diff --git a/NEWS.Rmd b/NEWS.Rmd index 7ad14192..35e155ea 100644 --- a/NEWS.Rmd +++ b/NEWS.Rmd @@ -3,10 +3,252 @@ output: rmarkdown::github_document --- +# SigRepo 1.1.0 + +## Breaking changes + +`runHypeR()` and `prepareHypeRSignatures()` now follow hypeR's own interface. + +- `runHypeR()` requires `organism` and `test`, with no default for either: + `runHypeR(omic_signature = sig, organism = "Homo sapiens", test = "hypergeometric", genesets = ...)`. + `organism` replaces `msigdb_species` (it is the MSigDB species with + `genesets = "msigdb"`), and a SigRepo signature whose `metadata$organism` is + different (ignoring case), or missing, is an error, so human and mouse + signatures run in separate calls. Before, the species silently defaulted to + `"Homo sapiens"` and mixed organisms were not detected. +- hypeR, msigdbr, fgsea, ggplot2, openxlsx and hypeR.GEM are now imported + rather than suggested, so they install with SigRepo. + +- `runHypeR()` returns hypeR's `hyp` or `multihyp` instead of + `list(result, signatures, metadata)`, and hypeR's functions (`hyp_dots()`, + `hyp_emap()`, `hyp_to_rmd()`, ...) work on it directly. The type follows the + input, as in hypeR: a `hyp` for a single signature (one `OmicSignature`, id + or name) with hypergeometric `split = FALSE`, otherwise a `multihyp`, even + when only one query is left. `hyp_show()` takes a single + `hyp`, e.g. `hyp_show(res$data[[1]])`. +- Each `hyp$info` ends with SigRepo provenance in a fixed order: + `SigRepo Signature ID`, `SigRepo Signature Name`, `Group Label`, + `Symbol Source`, `SigRepo Direction`, `SigRepo Ranked Table`, + `SigRepo Score Column`, `SigRepo Split`, `SigRepo Background Source`, + `SigRepo Features Unmapped`, `SigRepo Query Genes Removed`, + `SigRepo Genesets Dropped`, `SigRepo Genesets Dropped List`, `SigRepo FDR Scope`. +- `method` is now `test = c("hypergeometric", "kstest")`. The `"hypergeo"`, + `"ks"` and `"gsea"` aliases are gone; use `test = "kstest"` for a ranked test. + `power` changes only the kstest `score`: `1`, the default, weights hits by + |score|; `0` runs hypeR's ranked (unweighted) signature, so its `score` and + `Signature Type = "ranked"` match hypeR's documentation. The p-value and FDR + come from hypeR's unweighted, one-sided KS test and do not depend on `power`; + it finds genesets enriched toward the top of the ranking, which is why every + kstest also runs a `| down` query on the negated scores. +- kstest and fgsea always test both ends of the ranking; there is no + `direction` argument. kstest returns `"