diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..7bff9e56 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Editor configuration for this repository. The Julia sources indent with four +# spaces, the frontend with two, YAML with two, and every text file ends with +# a newline and no trailing whitespace. + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.jl] +indent_style = space +indent_size = 4 + +[*.{jl}] +max_line_length = off + +[{*.ts,*.tsx,*.js,*.jsx,*.json,*.css,*.html}] +indent_style = space +indent_size = 2 + +[{*.yml,*.yaml,*.kyaml}] +indent_style = space +indent_size = 2 + +[*.md] +indent_style = space +indent_size = 2 +# Markdown's hard line breaks are two trailing spaces; keep them. +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab + +[Justfile] +indent_style = space +indent_size = 4 diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..b02898a4 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,83 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement: the maintainer, [@JoshuaJewell](https://github.com/JoshuaJewell), using the contact details on their GitHub profile. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..f790ca1b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,57 @@ +name: Bug report +description: Something in MetaManifold does not work as documented. +labels: [bug] +body: + - type: markdown + attributes: + value: | + Search the existing issues first. Raise issues about a fork's own code on that fork. + - type: textarea + id: steps + attributes: + label: Steps or command + description: The command you ran, or the steps in the web interface, so that someone else can repeat them. + validations: + required: true + - type: textarea + id: expected + attributes: + label: What you expected + validations: + required: true + - type: textarea + id: actual + attributes: + label: What happened + description: Include the error message, the server log lines, or a screenshot. + validations: + required: true + - type: input + id: julia + attributes: + label: Julia version + placeholder: output of `julia --version` + validations: + required: true + - type: input + id: r + attributes: + label: R version + description: Only if the problem touches the R runtime; leave empty if you do not know it. + placeholder: output of `R --version | head -1` + - type: input + id: browser + attributes: + label: Browser and version + description: Only for problems in the web interface; leave empty for command-line reports. + - type: input + id: commit + attributes: + label: MetaManifold commit + placeholder: output of `git rev-parse --short HEAD` + - type: textarea + id: config + attributes: + label: Relevant configuration + description: The keys from `pipeline.yml` or the run's `run_config.yml` that bear on the problem. + render: yaml diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..312c2aec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,41 @@ +name: Feature request +description: Suggest a new feature or a change to an existing one. +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + Search the existing issues first. To change an architectural decision, describe the change + here before writing any code. The settled decisions in CONTRIBUTING.md can be discussed in an + issue, but a pull request does not reopen them. + - type: textarea + id: problem + attributes: + label: Problem + description: What you are trying to do, and what stops you now. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed change + description: What MetaManifold should do instead. For a statistical method, name the method and a reference for it. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + - type: dropdown + id: area + attributes: + label: Area + options: + - Pipeline stages + - Analysis and statistics + - Web interface + - Configuration + - Installation and toolchain + - Other + validations: + required: true diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 00000000..f97f2309 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,89 @@ + + +# Security policy + +## Deployment model, stated first + +MetaManifold is a **single-user, no-authentication** application: a Julia +server with a browser UI, designed to run on localhost or on a host trusted +by its one operator. The server carries real defences — a browser-origin +allowlist with a DNS-rebinding check, strict Host checking on loopback binds, +inert serving of uploaded report files, and Oxygen's docs and metrics pages +switched off — but **exposing it beyond its intended deployment is the +operator's risk, not a supported configuration.** Denial-of-service against +an internet-exposed instance is out of scope, as is anything that follows +from putting an unauthenticated API on a public network after ignoring the +deployment guidance in the README's *Deployment* section. + +## What is in scope + +- The HTTP surface: the origin allowlist and Host checks + (`src/server/server.jl`), the static-file and report-upload serving + (`_file_middleware`), SPA routing, and every route under + `src/server/routes/`. +- How configuration and primer values reach the external tools: pipeline + options and filenames become command lines; injection through them is a + finding here. +- Path handling: the file routes, report item paths, and study/run names as + path components. +- Installer and toolchain integrity: every download is SHA256-pinned + (`config/defaults/tool_versions.yml`); a tampered pin, an unchecked + substitute, or a way to make the installer fetch unverified content is a + finding. +- Provenance integrity: `attestation.yml` and the content hashes it records; + anything that can make a run attest to bytes it did not use. +- The dependency manifests: `Manifest.toml`, `renv.lock`, `frontend/bun.lock`. + +## What is out of scope + +- Vulnerabilities **in the external tools themselves** (cutadapt, FastQC, + MultiQC, DADA2, swarm, vsearch, cd-hit-est, MAFFT, trimAl, IQ-TREE, RAxML, + gappa, and the R packages): report those to their own projects. A way this + repository invokes them unsafely is in scope; their internal bugs are not. +- Julia, R, bun and DuckDB runtime CVEs: report upstream; this project pins + versions and will pick up fixes through its normal dependency updates. +- Internet-exposed DoS, as above. + +## Supported versions + +The default branch only. There are no backported releases; a fix lands on +`main` behind the usual CI gates. The version that matters for any report is +the git commit plus the pinned toolchain — and every run records exactly that +in its `attestation.yml`. + +## How to report + +**Preferred channel: private vulnerability reporting.** The repository has +it enabled; the *Report a vulnerability* button on the Security tab reaches +the maintainers privately. Use it. + +**By email:** contact **developer@joshuajewell.dev** (Jonathan Jewell, the project +owner) with `[security]` in the subject line. + +Include, where they apply: + +- what you found, and the impact you think it has; +- the endpoint, route, or pipeline stage it affects; +- the deployment mode — localhost, SSH tunnel, or split frontend/API; +- reproduction steps, or the request that triggers it; +- the versions block from the affected run's `attestation.yml` — it names the + exact commit and toolchain, which usually settles the first two replies. + +Reports are acknowledged when triaged, fixed on `main` with the normal CI +gates, and disclosed on the reporter's preferred schedule. Credit is offered +unless you prefer otherwise. + +## Mathematical foundations + +This project carries machine-checked proofs of some of its statistics, and it +treats **unsound mathematics as a reportable weakness of its own kind**: a +proof that is vacuous, a theorem that does not say what the code relies on, or +a statistical claim the arithmetic does not support. If you find one, that is +a finding — send it to **j.d.a.jewell@open.ac.uk**, where the +proof-theoretic side of the project lives. Questions and open debate about +the foundations belong in the +[mathematical foundations discussion](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/149); +private concerns about soundness belong in a report. diff --git a/codecov.yml b/.github/codecov.yml similarity index 100% rename from codecov.yml rename to .github/codecov.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..2f06b99e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,53 @@ +# Dependabot version updates. Until this file landed, only security updates +# ran (those need no configuration), and those failed for a separate reason: +# the "Dependabot Updates" runs of 2026-09-26..27 tried to fix alerts against +# fixtures under test/doi using the npm_and_yarn updater, which refuses a +# Bun-managed project ("bun.lock but no package-lock.json"). Those fixtures +# have since left the repository and both alerts are fixed. +# +# What cannot be covered here: Julia (Project.toml / Manifest.toml) and R +# (renv.lock) are not Dependabot package-ecosystems. Their pins live in +# config/defaults/tool_versions.yml and are asserted by +# test/unit/test_install_pins.jl and the CI workflow instead. +# +# The workflows pin every action to a full commit SHA (this account's +# allowlist policy, asserted by test/unit/test_install_pins.jl). Dependabot +# advances those pins to the current SHA of the tracked version, so the +# pinning policy itself survives updates. +# +# The #124 rule: minor and patch updates are batched, because they are +# interchangeable in kind; a MAJOR (React 18->19, vite 6->8, TypeScript +# 5->7 -- which #124 landed as one grouped PR, with the build already red +# until #140 repaired it) is a migration, not an update. Every group below +# therefore names its update-types as minor and patch ONLY: an update that +# matches no group opens its own PR, carrying its own review and its own CI +# run. test/unit/test_install_pins.jl asserts this shape, so a new group +# cannot silently go back to batching majors. +version: 2 +updates: + # The frontend is Bun-managed: package.json plus bun.lock, no npm/yarn/pnpm + # lockfile, so it must be declared as the "bun" ecosystem. + - package-ecosystem: "bun" + directory: "/frontend" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(deps)" + groups: + # Minor and patch in one weekly PR; majors arrive one PR each. + frontend-minor-and-patch: + patterns: ["*"] + update-types: ["minor", "patch"] + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(deps)" + groups: + # Same rule as the frontend: a new major of an action (checkout + # v4.4.0 -> v7.0.1 came through #123) gets a PR of its own. + actions-minor-and-patch: + patterns: ["*"] + update-types: ["minor", "patch"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..a17d0de6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,43 @@ +## Summary + + + +Closes # + +## Type of change + +- [ ] 🐛 Bug fix (non-breaking change that fixes an issue) +- [ ] ✨ New feature (non-breaking change that adds functionality) +- [ ] 💥 Breaking change (would change existing behaviour, a config key or an API response) +- [ ] 📊 Statistics fix (a method returned a wrong number, or a result it could not support) +- [ ] 📖 Documentation +- [ ] 🧹 Refactor / tech debt (behaviour-preserving) +- [ ] ⚡ Performance +- [ ] 🔧 Build / CI / tooling + +## How has this been verified? + + + +Tested with Julia `…`, R `…` and bun `…`. + +## What this leaves out + + + +## Checklist + +- [ ] My branch starts at the current tip of `main` (`behind_by` is `0`; see CONTRIBUTING.md) and has no merge commits from `main`. +- [ ] I ran `julia --project=. test/runtests.jl` (and `cd frontend && bun run test` for frontend changes) and they pass. +- [ ] CI ran on this PR and passes. +- [ ] No generated or vendored files, and no lockfile change the work does not need (or the reason is given above). +- [ ] New code is AGPL-3.0, or MPL-2.0 with an SPDX identifier in each file I wrote. I left existing file headers, `LICENSE`, `CITATION.cff` and the licence and acknowledgement sections of `README.md` unchanged. +- [ ] Every number shown to a user is computed from their data. When a method cannot run it raises an explicit error. +- [ ] Docs are updated, and no claim in them overstates what the code does. + +## Notes for reviewers + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10cf5483..b5f4cd1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,20 @@ name: CI +# This fork's account allows only GitHub's own actions (actions/*) and its own, +# each pinned to a full commit SHA. Upstream's julia-actions, oven-sh/setup-bun +# and codecov/codecov-action are none of these, and with them this workflow +# ended in startup_failure on every run. So Julia and Bun are installed here +# from the archives config/defaults/tool_versions.yml pins, each refused unless +# it hashes to the pinned SHA256, and the Julia depot is cached with +# actions/cache. Coverage is kept as a workflow artifact, since the Codecov +# project is upstream's. test/unit/test_install_pins.jl asserts that every +# `uses:` in .github/workflows is allowed by that policy. + on: push: branches: [main] + # Any base branch, so a PR stacked on another one is tested too. pull_request: - branches: [main] permissions: contents: read @@ -16,6 +26,95 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'push' }} jobs: + # A pull request that touches nothing the suite exercises -- a README edit, a + # wiki page, a template, a licence record -- does not need the ~19-minute + # Julia suite or the proof checker to pass over it. The gate decides, and the + # two jobs below carry `if:` on its output; when it says not relevant, GitHub + # posts `skipped` for them, and a skipped check satisfies a required one. The + # gate itself must never be skipped -- it is the check that vouches for the + # others' absence, so Branch-Floor requires it too. + # + # The trigger stays unfiltered for a reason this repository has already + # sprang once (#116): a path-filtered `pull_request` trigger does not run at + # all on a PR that matches no path, so a required check from that workflow + # never posts and can never be satisfied. Running the cheap gate on every PR + # and letting it skip the heavy work is the pattern the occupancy workflow + # (#125) established; this applies it to the main suite. + # + # The relevant-path list is deliberately the positive form: anything not + # listed is not relevant, so a new directory joins the suite by being added + # here, not by being forgotten. These tests also read the repository's own + # shape -- the pin tests parse `.github/workflows/`, the root-shape guard + # reads the tracked tree, and the occupancy reading's Agda tree lives under + # `docs/occupancy/` -- so those paths are relevant even though they are + # documentation-shaped. In a `case` pattern, `*` crosses `/`. + gate: + name: Code relevance + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} + steps: + # A push only starts this workflow on main (the branch filter above), so + # the gate has nothing to decide: every push is relevant. + - name: A push is already branch-filtered + id: push + if: github.event_name == 'push' + run: echo 'relevant=true' >> "$GITHUB_OUTPUT" + + # Full history, because the three-dot diff names the merge base of the + # PR's base tip and its merge commit, and a stacked PR's merge base sits + # further back still. + - name: Checkout the change + if: github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # nothing here pushes + fetch-depth: 0 + + # --no-renames, so a file renamed out of a listed path is reported as a + # deletion under its original name and still counts. -z with a + # NUL-delimited read, so a pathname Git would quote or that contains a + # newline arrives raw and matches the pattern. The base SHA travels + # through env, not template interpolation into the script. + - name: Does the change touch anything the suite exercises? + id: pr + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + relevant=false + # Every pattern is quoted, so the glob survives array assignment and + # fires at `case` time, against the diff -- an unquoted `src/*` would + # expand against the checkout the runner happens to be standing in. + patterns=('src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*' + 'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml' + 'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' + 'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc' + '.github/workflows/*') + # The diff goes to a file first because a process substitution hides + # git diff's exit status: an unavailable comparison SHA (a force-push + # on the base branch, for instance) or a repository error would read + # as "no changes", and the gate would post relevant=false -- skipping + # the Julia suite and the proof checker, whose `skipped` conclusion + # satisfies a required check -- so a relevant PR would pass silently. + # A failed diff must fail the gate itself: that is the one failure + # the required-check design cannot afford to skip. As in the + # occupancy gate, which this mirrors. + changed="$RUNNER_TEMP/ci-changed-files" + if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then + echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 + exit 1 + fi + while IFS= read -r -d '' file; do + for pattern in "${patterns[@]}"; do + case "$file" in + $pattern) relevant=true; break 2 ;; + esac + done + done < "$changed" + echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + test: # The name is a fixed string, it carries no version, and the job has NO matrix. # Both are required, and the first without the second is a trap that has already @@ -40,6 +139,10 @@ jobs: # silently. If a matrix is ever genuinely wanted here, the required context must move # to a matrix-free aggregator job first. name: Julia tests + # Runs only when the gate above found a relevant path; on a docs-only PR + # this job posts `skipped`, which satisfies the required check. + needs: gate + if: needs.gate.outputs.relevant == 'true' # Explicit, not ubuntu-latest: the R pin below names an apt package revision built # for 24.04, and a runner that silently rolled to the next LTS would take that pin # with it. `runs-on` cannot read `env`, so this is a literal. @@ -53,22 +156,45 @@ jobs: # the environment, so those commands pass --no-init-file instead. env: RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" + # Pinned rather than tracking latest stable, because a pinned Manifest.toml + # resolved by an unpinned Julia is not a reproducible build. These must equal + # runtimes.julia.archives.linux-x86_64 in the pin file, whose version must + # equal julia_version in Manifest.toml; they cannot be read from the pin + # file, since nothing can be read before Julia exists. + # test/unit/test_install_pins.jl fails if any of them disagree. + JULIA_URL: https://julialang-s3.julialang.org/bin/linux/x64/1.12/julia-1.12.5-linux-x86_64.tar.gz + JULIA_SHA256: 41b84d727e4e96fbf3ed9e92fa195d773d247b9097f73fad688f8b699758bae7 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - name: Set up Julia - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # Pinned rather than tracking latest stable, because a pinned Manifest.toml - # resolved by an unpinned Julia is not a reproducible build. This must equal - # julia_version in Manifest.toml; it cannot be read from the pin file, since - # nothing can be read before Julia exists. test/unit/test_install_pins.jl - # fails if the three ever disagree. - version: "1.12.5" + persist-credentials: false # nothing here pushes - - name: Cache Julia packages - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0 + - name: Install Julia + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$JULIA_URL" julia.tar.gz + echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - + sudo mkdir -p /opt/julia + sudo tar xzf julia.tar.gz -C /opt/julia --strip-components=1 + rm julia.tar.gz + echo /opt/julia/bin >> "$GITHUB_PATH" + + # The registry, packages, artifacts and precompiled caches, keyed on the + # Manifest (which also names the Julia version). Saved once instantiation + # has succeeded, so a failed run cannot store a half-built depot. + - name: Restore the Julia depot + id: julia-depot + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.julia/registries + ~/.julia/packages + ~/.julia/artifacts + ~/.julia/compiled + key: julia-depot-${{ runner.os }}-${{ hashFiles('Manifest.toml') }} + restore-keys: | + julia-depot-${{ runner.os }}- # Every external version CI installs is read from the committed pin file, so CI # and a developer's machine cannot drift apart. A temporary environment is used @@ -84,7 +210,10 @@ jobs: pins = YAML.load_file("config/defaults/tool_versions.yml") open(ENV["GITHUB_ENV"], "a") do io println(io, "R_APT_VERSION=", pins["runtimes"]["r"]["apt_version"]) - println(io, "BUN_VERSION=", pins["toolchain"]["bun"]["version"]) + bun = pins["toolchain"]["bun"] + println(io, "BUN_VERSION=", bun["version"]) + println(io, "BUN_URL=", bun["archives"]["linux-x86_64"]["url"]) + println(io, "BUN_SHA256=", bun["archives"]["linux-x86_64"]["sha256"]) println(io, "CUTADAPT_SPEC=cutadapt==", pins["tools"]["cutadapt"]["version"]) println(io, "MULTIQC_SPEC=multiqc==", pins["tools"]["multiqc"]["version"]) fastqc = pins["tools"]["fastqc"]["archives"]["any"] @@ -139,7 +268,7 @@ jobs: # the restore runs as root. The run id makes each key unique so every attempt # saves its progress; restore-keys picks the newest match. - name: Restore the renv cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /opt/renv-cache key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }} @@ -148,6 +277,12 @@ jobs: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}- # R_CLI_DYNAMIC and TERM make renv print progress one line at a time in the log. + # This verification list -- and the RCall visibility check below -- must + # track renv.lock: restore() installs only what the lock records, so a + # package removed from the lock fails here at verification, nowhere near + # the code that stopped using it. MEASURED in this PR: dplyr left the lock + # and this check failed until both lists were updated. Keep them in the + # same commit as any lock change. - name: Install R packages env: RENV_PATHS_CACHE: /opt/renv-cache @@ -156,7 +291,7 @@ jobs: sudo chmod 777 "$RENV_PATHS_CACHE" sudo env R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'install.packages(c("renv", "BiocManager"), repos="https://cloud.r-project.org", lib=.Library)' sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)' - Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","dplyr","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' + Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' # Saved on failure too, since the packages built before a failure are the ones # the next attempt should reuse. Root writes the cache, so it is made readable first. @@ -166,7 +301,7 @@ jobs: - name: Save the renv cache if: always() - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /opt/renv-cache key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }} @@ -221,10 +356,28 @@ jobs: - name: Instantiate Julia project run: julia --project=. -e 'import Pkg; Pkg.instantiate()' - - name: Set up Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + - name: Save the Julia depot + if: steps.julia-depot.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - bun-version: ${{ env.BUN_VERSION }} + path: | + ~/.julia/registries + ~/.julia/packages + ~/.julia/artifacts + ~/.julia/compiled + key: julia-depot-${{ runner.os }}-${{ hashFiles('Manifest.toml') }} + + # Downloaded against the pinned URL and refused unless it hashes to the pinned + # SHA256, as install.jl does when no bun of this version is on PATH. + - name: Install Bun + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$BUN_URL" bun.zip + echo "$BUN_SHA256 bun.zip" | sha256sum -c - + unzip -q bun.zip + sudo mv bun-linux-x64/bun /usr/local/bin/bun + rm -rf bun.zip bun-linux-x64 + test "$(bun --version)" = "$BUN_VERSION" - name: Build frontend working-directory: frontend @@ -243,11 +396,13 @@ jobs: run: julia --project=. -e 'import Pkg; Pkg.build("RCall")' - name: Verify R packages visible from RCall + # The second of the two lists that must track renv.lock; see the + # Install R packages step above for the coupling. run: | julia --project=. -e ' using RCall R"cat(.libPaths(), sep=\"\n\")" - for pkg in ["dada2", "dplyr", "vegan", "tibble", "MASS", "zCompositions"] + for pkg in ["dada2", "vegan", "tibble", "MASS", "zCompositions"] R"if (!require($pkg, character.only=TRUE, quietly=TRUE)) stop($pkg, \" not found\")" end ' @@ -289,19 +444,29 @@ jobs: fi exit "$status" + # What julia-processcoverage did: the .cov files under src/ to lcov.info, with + # CoverageTools pinned to an exact release. - name: Process coverage - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1.2.2 - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2 + run: | + julia -e ' + using Pkg + Pkg.activate(temp = true) + Pkg.add(name = "CoverageTools", version = "1.4.2") + using CoverageTools + LCOV.writefile("lcov.info", process_folder("src"))' + + # The Codecov project (codecov.yml, the README badge) is upstream's, so the fork + # keeps its coverage as an artifact of the run instead. + - name: Keep coverage + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - files: lcov.info - token: ${{ secrets.CODECOV_TOKEN }} - slug: JoshuaJewell/MetaManifold-WebUI - fail_ci_if_error: false + name: lcov + path: lcov.info + retention-days: 14 - # Machine-checked proofs of the Benjamini-Hochberg adjustment and the TSS/RLE - # scaling algebra (proofs/agda, proofs/agda/README.md). + # Machine-checked proofs of the Benjamini-Hochberg adjustment, the TSS/RLE + # scaling algebra, PERMANOVA's permutation bookkeeping and the significance + # reporting discipline (proofs/agda, proofs/agda/README.md). # Agda publishes no 2.6.4.3 binary, and agda-stdlib 2.1 requires Agda 2.6.4.x, # so the pair comes from Debian trixie, which packages exactly that pair with # pre-built library interfaces. The image is pinned by digest and the two @@ -311,13 +476,18 @@ jobs: # reads them. agda: name: Agda proofs + # As the Julia suite above: only when the gate found a relevant path. + needs: gate + if: needs.gate.outputs.relevant == 'true' runs-on: ubuntu-24.04 container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb timeout-minutes: 30 env: DEBIAN_FRONTEND: noninteractive steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # nothing here pushes - name: Install Agda and the standard library run: | diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 00000000..c59ba873 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,46 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, ready_for_review, reopened] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + if: github.event.pull_request.head.repo.full_name == github.repository + # Optional: Filter by PR author + # if: | + # github.event.pull_request.user.login == 'external-contributor' || + # github.event.pull_request.user.login == 'new-developer' || + # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@374f254f5efaf968abfe15d399fd90972e8138b9 # v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review --comment ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + claude_args: '--allowedTools "mcp__github_inline_comment__create_inline_comment"' + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 00000000..c699b80f --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@374f254f5efaf968abfe15d399fd90972e8138b9 # v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr *)' + diff --git a/.github/workflows/occupancy.yml b/.github/workflows/occupancy.yml new file mode 100644 index 00000000..0068dab5 --- /dev/null +++ b/.github/workflows/occupancy.yml @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# The occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): +# * the two-thread disagreement test (test/occupancy_disagreement_test.jl) on +# the pinned vsearch, and +# * Session IR's counted bytes against the typed peak computed in Agda from +# the pinned proof siblings (scripts/check-occupancy.sh). +# +# This account allows only GitHub's own actions and its own, pinned by SHA, so +# everything here installs from pinned URLs checked against pinned SHA256s, as +# ci.yml now does too. The disagreement test also runs in ci.yml's full suite; +# it runs here as well so that a change to the probe gets its own check. +# +# #116 merged with this workflow's "Occupancy typed peak" check red (its squash +# carried a draft `TH.[_]` placeholder; #117 fixed the proof), because nothing +# required the check to pass first -- and nothing COULD have: a check posts +# only when its workflow runs, and the pull_request trigger was path-filtered, +# so requiring these checks would have deadlocked every PR that touches no +# occupancy file. A required check that never posts can never be satisfied; the +# trap ci.yml's header records for its own job names. +# +# So the pull_request path filter moved into the `Occupancy relevance` gate +# below. The workflow now runs on EVERY pull request; the gate decides whether +# the change touches the occupancy reading; the two heavy jobs run only when it +# does, and post their checks as `skipped` otherwise, which a required check +# counts as satisfied. The gate itself posts on every PR too, and it is the one +# check of the three that must be required without exception: if the gate +# failed, its dependents would be skipped and would satisfy their requirement +# silently. The push trigger keeps its native paths filter, so a push to main +# still starts this workflow only when it should. +name: Occupancy reading + +on: + push: + branches: [main] + paths: + - src/core/memory_probe.jl + - src/core/provenance.jl + - src/pipeline/tools.jl + - src/server/routes/pipeline.jl + - test/occupancy_disagreement_test.jl + - test/unit/test_occupancy_grade.jl + - scripts/check-occupancy-grades.jl + - scripts/lint-occupancy-readings.sh + - docs/occupancy/** + - docs/occupancy-2026-10.adoc + - scripts/check-occupancy.sh + - scripts/fetch-siblings.sh + - .github/workflows/occupancy.yml + # No paths filter, on purpose (see above): the gate must post its check on + # every PR, whichever files the PR touches. Any base branch, so a PR stacked + # on another one is gated too. + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'push' }} + +env: + # Julia 1.12.5, the version Manifest.toml was resolved with. The checksum is + # julialang's own (bin/checksums/julia-1.12.5.sha256), as the official Docker + # image (docker-library/julia a39df01) records it. + JULIA_URL: https://julialang-s3.julialang.org/bin/linux/x64/1.12/julia-1.12.5-linux-x86_64.tar.gz + JULIA_SHA256: 41b84d727e4e96fbf3ed9e92fa195d773d247b9097f73fad688f8b699758bae7 + +jobs: + # Decides whether the PR touches the occupancy reading at all. The file list + # is the pull_request half of the old paths filter, moved here because a + # pinned-actions workflow cannot use a third-party paths-filter action (the + # account's allow-list refuses one) -- plain `case` globs are all it takes. + # In a case pattern `*` crosses `/`, so `docs/occupancy/*` is the old + # `docs/occupancy/**`. + gate: + name: Occupancy relevance + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} + steps: + # A push only starts this workflow when a `paths:` file changed (the + # trigger filter above), so the gate has nothing to decide on pushes. + - name: A push is already path-filtered + id: push + if: github.event_name == 'push' + run: echo 'relevant=true' >> "$GITHUB_OUTPUT" + + # Full history, because the three-dot diff names the merge base of the + # PR's base tip and its merge commit -- a shallow checkout does not have + # both, and a stacked PR's merge base sits further back still. + - name: Checkout the change + if: github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # nothing here pushes + fetch-depth: 0 + + - name: Does the change touch the occupancy reading? + id: pr + if: github.event_name == 'pull_request' + env: + # zizmor flags template expressions interpolated into run scripts + # (template-injection), and the gate script is easier to run locally + # with the value in an environment variable in any case. The base SHA + # is GitHub-supplied, but the indirection costs nothing. + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + relevant=false + patterns=(src/core/memory_probe.jl src/core/provenance.jl src/pipeline/tools.jl + src/server/routes/pipeline.jl test/occupancy_disagreement_test.jl + test/unit/test_occupancy_grade.jl scripts/check-occupancy-grades.jl + scripts/lint-occupancy-readings.sh 'docs/occupancy/*' + docs/occupancy-2026-10.adoc scripts/check-occupancy.sh + scripts/fetch-siblings.sh .github/workflows/occupancy.yml) + # The diff goes to a file first because a process substitution hides + # git diff's exit status: an unavailable comparison SHA or a repository + # error would read as "no changes", and the gate would post + # relevant=false -- skipping the two occupancy jobs, whose `skipped` + # conclusion satisfies a required check -- so a relevant PR would + # pass silently. A failed diff must fail the gate itself: that is + # the one failure the required-check design cannot afford to skip. + changed="$RUNNER_TEMP/occupancy-changed-files" + if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then + echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 + exit 1 + fi + while IFS= read -r -d '' file; do + for pattern in "${patterns[@]}"; do + case "$file" in + $pattern) relevant=true; break 2 ;; + esac + done + done < "$changed" + echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + + disagreement: + needs: gate + if: needs.gate.outputs.relevant == 'true' + name: Occupancy disagreement test + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + # As in ci.yml: the root .Rprofile would otherwise bootstrap renv. + RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Julia + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$JULIA_URL" julia.tar.gz + echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - + sudo mkdir -p /opt/julia + sudo tar xzf julia.tar.gz -C /opt/julia --strip-components=1 + rm julia.tar.gz + echo /opt/julia/bin >> "$GITHUB_PATH" + + # RCall builds against R. The occupancy test uses no R package. + - name: Install R + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends r-base-core + + - name: Instantiate the project + run: julia --project=. -e 'using Pkg; Pkg.instantiate()' + + # The pinned vsearch (config/defaults/tool_versions.yml), refused unless it + # hashes to the pinned SHA256, as ci.yml installs it. + - name: Install vsearch + run: | + eval "$(julia --project=. -e ' + using YAML + v = YAML.load_file("config/defaults/tool_versions.yml")["tools"]["vsearch"] + a = v["archives"]["linux-x86_64"] + println("VSEARCH_VERSION=", v["version"]); println("VSEARCH_URL=", a["url"]); println("VSEARCH_SHA256=", a["sha256"])')" + source scripts/ci/fetch_pinned.sh + fetch_pinned "$VSEARCH_URL" vsearch.tar.gz + echo "$VSEARCH_SHA256 vsearch.tar.gz" | sha256sum -c - + tar xzf vsearch.tar.gz --warning=no-unknown-keyword + sudo mv "vsearch-$VSEARCH_VERSION-linux-x86_64/bin/vsearch" /usr/local/bin/vsearch + rm -rf vsearch.tar.gz "vsearch-$VSEARCH_VERSION-linux-x86_64" + vsearch --version + + - name: Three orders of the vsearch stage + run: julia --project=. -t 2 test/occupancy_disagreement_test.jl + + # Fork issue #105: the grades the recorded run's windows imply, read back + # against its own measurement (docs/occupancy/schedules.yml). + - name: Graded reading of the recorded run + run: julia --project=. scripts/check-occupancy-grades.jl + + # And no production code reads a peak without those grades. + - name: Occupancy readings are graded + run: bash scripts/lint-occupancy-readings.sh + + typed-peak: + needs: gate + if: needs.gate.outputs.relevant == 'true' + name: Occupancy typed peak + runs-on: ubuntu-24.04 + # The Agda 2.6.4.3 Debian trixie packages, in a digest-pinned image, as the + # upstream proof job uses. The siblings need no standard library. + container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb + timeout-minutes: 20 + env: + DEBIAN_FRONTEND: noninteractive + steps: + # git before checkout, so the checkout is a repository and the siblings can be fetched. + - name: Install git, curl and Agda + run: | + apt-get update -qq + apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 + agda --version | grep -qx 'Agda version 2.6.4.3' + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # the siblings are public; nothing here pushes + + # The Session IR stepper is Julia (standard library only). + - name: Install Julia + shell: bash + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$JULIA_URL" julia.tar.gz + echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - + mkdir -p /opt/julia + tar xzf julia.tar.gz -C /opt/julia --strip-components=1 + rm julia.tar.gz + echo /opt/julia/bin >> "$GITHUB_PATH" + + - name: Counted bytes against typed peak + shell: bash + run: bash scripts/check-occupancy.sh diff --git a/.gitignore b/.gitignore index 6ef7e458..421b96ab 100644 --- a/.gitignore +++ b/.gitignore @@ -250,6 +250,9 @@ vignettes/*.pdf ## Documentation kept local except published release notes docs/* !docs/release-notes/ +# and the dated occupancy reading (MMW-01), whose table the gate checks +!docs/occupancy-2026-10.adoc +!docs/occupancy/ # translation temp files @@ -320,6 +323,7 @@ archive/ !.github !.github/** !.Rprofile +!.editorconfig # Brainstorming visual-companion scratch (mock-ups, server state) .superpowers/ @@ -335,3 +339,6 @@ dev/server.log # Agda interface files (proofs/agda) proofs/agda/_build/ +# Proof siblings fetched by scripts/fetch-siblings.sh, and Agda interface files +/_build/ +*.agdai diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..d4164769 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# AGENTS.md + +Instructions for coding agents working in this repository. Read the contribution +rules in `.github/CONTRIBUTING.md` first; this file records only the traps that +are invisible until they bite. + +## renv.lock and CI's hardcoded R-package lists are coupled + +CI installs R from `renv.lock` (`renv::restore()`), but `.github/workflows/ci.yml` +names the expected packages in TWO hardcoded verification lists: + +- the `Install R packages` step's `require()` check ("failed to install"), and +- the `Verify R packages visible from RCall` step. + +Removing a package from `renv.lock` therefore fails the run at install +verification, nowhere near the code that stopped using it. MEASURED on +JoshuaJewell/MetaManifold-WebUI#57 (2026-10-15): dplyr left the lock and both +checks failed until the lists were updated in the same PR. Whenever a change +touches `renv.lock`, grep `.github/workflows/ci.yml` for every added or removed +package name and update both lists in the same commit. + +Corollary, measured the same way: `tibble` cannot leave the lock while dada2 is +in it. `dada2 -> ggplot2 -> tibble` is dada2's own dependency chain, independent +of anything this project's code does; the lock's `Requirements` arrays are not +evidence (only a handful of records carry one). + +## Commit signing needs the SSH agent, not a passphrase prompt + +`commit.gpgsign` is on, with a passphrase-protected SSH key. A non-interactive +shell has no askpass and no terminal for a prompt, so signing fails with a +misleading "failed to write commit object" unless the agent is reachable: +`SSH_AUTH_SOCK=/tmp/lcb-ssh-agent.sock` (the fixed socket; `~/.profile` exports +it when the socket exists). Never work around this with +`commit.gpgsign=false` — check the agent first with +`SSH_AUTH_SOCK=/tmp/lcb-ssh-agent.sock ssh-add -l`. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..9f52ed7f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ + + +# Changelog + +Release notes live under `docs/release-notes/`, one file per release: + +- [v0.1.0](docs/release-notes/v0.1.0.md) + +This file is the root index the RSR scheme expects; the notes themselves are +kept where they have always been, and a new release adds a line here in the +same commit that adds its file there. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..b02898a4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,83 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement: the maintainer, [@JoshuaJewell](https://github.com/JoshuaJewell), using the contact details on their GitHub profile. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..b1cbe4da --- /dev/null +++ b/Justfile @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# MetaManifold's task entry (RSR scheme: a thin Justfile at the root; the +# heavy work lives in the commands it names). Every recipe delegates to the +# repository's own documented entry points -- install.sh, start.sh, the test +# runner -- so there is exactly one place where each of those is defined and +# this file never re-implements them. + +# The default recipe lists what exists; `just --list` does the same. +default: + @just --list + +# The documented installer: Julia (pinned), R, the external tools, the +# frontend build. Extra args pass through to install.sh (--update, +# --modify, --sysimage). +install *ARGS: + bash install.sh {{ARGS}} + +# The documented server entry point. Serves the UI on port 8080. +serve: + bash start.sh + +# The full unit suite. Name files to run a subset: +# just test test_install_pins.jl test_server_security.jl +test *FILES: + julia --project=. test/runtests.jl {{FILES}} + +# The pin suite alone: toolchain pins, lockfile agreement, workflow shape, +# and the root-shape allowlist. +pins: + julia --project=. test/runtests.jl test_install_pins.jl + +# The Agda proof tree's audit (no postulates, no holes, everything reached +# from All.agda). The type-check itself runs in CI's Agda proofs job. +proofs: + bash proofs/tests/axiom-audit.sh + +# The frontend's own checks. +frontend-test: + cd frontend && bun run test + +frontend-typecheck: + cd frontend && bun run typecheck + +frontend-build: + cd frontend && bun run build + +# The git-signing agent's state, as the launcher sees it. +agent-status: + ./launcher.sh --agent status diff --git a/R/_renv_dependencies.R b/R/_renv_dependencies.R index d9b4c570..1746380f 100644 --- a/R/_renv_dependencies.R +++ b/R/_renv_dependencies.R @@ -3,7 +3,5 @@ if (FALSE) { library(dada2) library(vegan) - library(dplyr) - library(tibble) library(MASS) } diff --git a/README.md b/README.md index 1fe02ca8..db9a5745 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,16 @@ Once a run completes, analysis is performed on request through the web UI, both - Pipeline stage read-count summaries - NMDS ordination (Bray-Curtis, via R/vegan) - PERMANOVA, with PERMDISP to check for differences in dispersion (via R/vegan) +- Differential abundance between two conditions: a negative-binomial model per taxon (via R/MASS) with Benjamini-Hochberg adjusted p-values (see [Differential abundance](#differential-abundance)) Counts may be normalised before analysis (none, rarefaction to a fixed or auto-resolved depth, or scaling with ranked subsampling (SRS), which reaches the depth by scaling counts and keeps more of the community structure), and contamination-flagged taxa may be included or excluded. Ordinations can also apply a Hellinger transform before the Bray-Curtis dissimilarity. Composition charts can be drawn as a grid faceted on any two of run, group and sub-group. All analysis charts are returned as Plotly JSON and rendered interactively in the browser. +## Documentation + +Longer documentation for users, maintainers, developers and theorists is kept as a wiki in +[metadatastician/MetaManifold-Evidence, under `docs/wikis/`](https://github.com/metadatastician/MetaManifold-Evidence/tree/main/docs/wikis). +Start at its [Home page](https://github.com/metadatastician/MetaManifold-Evidence/blob/main/docs/wikis/Home.md). + ## Prerequisites - **Julia** 1.12 (installed by `install.sh` if missing, and pinned to the Manifest version for this directory) @@ -94,7 +101,7 @@ bash install.sh ``` `install.sh` installs Julia (via juliaup) if missing, checks for R, then hands -off to `install.jl`, which installs the Julia dependencies, locates or downloads +off to `scripts/install.jl`, which installs the Julia dependencies, locates or downloads each external tool (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est, and the phylogeny tools MAFFT, trimAl, IQ-TREE, RAxML and gappa), and reproduces the R environment from `renv.lock`. It ends with a summary listing every dependency as @@ -180,6 +187,18 @@ precompile_workload = false The server then makes those requests itself in the background as it starts, so the first page visit does not wait on compilation. +## Community + +[Discussions](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions) are open, and they are where the tool's direction is argued out: + +- [Introduce yourself](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/148) — what you run through MetaManifold, and how it went. +- [Ideas](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/150) — what the tool should do next; small conveniences and whole methods both welcome. +- [The mathematical foundations thread](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/149) — the statistics underneath the analyses: what is proved, what is questionable, what deserves a proof. +- [Show and tell](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/151) — figures, publication tables and placement trees that came out of a run. +- [Q&A](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/152) — setup, usage and interpretation. A reproducible defect is a bug report, not a Q&A. + +One house norm: runs record their exact toolchain in `attestation.yml`; paste its versions block when asking why results differ. + ## Web interface The browser interface is the primary way to drive MetaManifold. Beyond creating studies and launching runs, it offers the following: @@ -240,6 +259,80 @@ The composition view classifies each ASV/OTU into a biological category and rend Organism-category composition of two runs as stacked bars

+### Differential abundance + +The Differential Abundance tab of a study's analysis workspace tests each taxon +for a difference in abundance between two conditions. Select exactly two runs or +pooled-run sub-groups, a results table and a rank. The first condition is the +reference, so a positive fold change means the taxon is more abundant in the +second. The rank list shows only the ranks the selected runs share. With +aggregation on, a pooled run counts as one condition rather than one per +sub-group. + +Counts are summed per taxon at the chosen rank, with rows that have no name at +that rank pooled as `Unclassified`. This happens after the read-count bounds and +any `analysis.exclude_categories` entry that applies to `differential`. Each +taxon is fitted by the configured method. The default, `nb_glm`, fits a +negative-binomial GLM, `MASS::glm.nb(y ~ group + offset(log(size factor)))`, +on the raw integer read counts; there are no pseudocounts, the group +coefficient gets a Wald test, and the effect is a log2 fold change. Size +factors come from the whole count matrix and are centred to a geometric mean +of 1. The alternative, `clr_welch`, runs Welch's t-test on centred log-ratios +over the taxa passing `min_prevalence`, after `zCompositions::cmultRepl` +replaces their zeros; its effect is a difference in CLR units, not a fold +change. Under either method, Benjamini-Hochberg adjustment runs over the taxa +that produced a p-value. The code is in +`src/analysis/differential.jl`, and the route in `src/server/routes/analysis.jl`. + +Settings live under `analysis.differential` (see +[Configuring analysis](#configuring-analysis-analysis-in-pipelineyml)): + +- `method`: `nb_glm` (default) or `clr_welch`, as above. +- `offset`: `tss` (default) uses each sample's total reads. `rle` uses the + median of ratios over the taxa with reads in every sample. Used by `nb_glm` + only; `clr_welch` ignores it, since log-ratios carry no library size. +- `min_prevalence`: a fraction from 0 to 1 (default 0). A taxon with reads in a + smaller fraction of the samples is not fitted. For `clr_welch` it is applied + before zero replacement, so on a sparse table it also keeps imputed values + for rare taxa out of every sample. +- `replacement_delta`: `clr_welch` only; `cmultRepl`'s `frac`, between 0 and 1 + (default 0.65). An imputed proportion above its taxon's smallest observed + proportion is lowered to this fraction of it. A replaced zero is not a + measurement. + +The result is a volcano plot and a table. The plot draws log2 fold change +against -log10 of the raw p-value, coloured by whether the adjusted p-value is +below 0.05. Taxa without an adjusted p-value are left off the plot but stay in +the table. Each table row has a status: + +- `ok`: the model fitted. +- `boundary`: the model fitted, but the dispersion parameter theta reached a + bound (at least 1e7, so effectively Poisson, or at most 1e-8). These taxa + keep their p-value and stay in the adjusted family. +- `failed`: no p-value, with the reason. Causes are counts that are constant + across samples, an error from `glm.nb`, a fit that did not converge, an + aliased group coefficient, or a non-finite estimate. +- `filtered`: below `min_prevalence`, so not fitted. + +A failed or filtered taxon never gets a stand-in p-value, and it is not counted +in the adjustment. The table downloads as `differential_abundance.csv` and can +be added to the report. The CSV starts with `#` lines that record the +comparison, method and settings. + +The test refuses, with an explicit error, when: + +- the selection is not exactly two conditions, or both are the same; +- the two conditions resolve to different ranks; +- a condition has no results table, no taxonomy columns or no sample columns, + or the read-count bounds remove all its samples; +- `offset` or `min_prevalence` is invalid; +- a count is not a non-negative integer, a condition has no samples, or there + are fewer than 3 samples in total; +- `tss` meets a sample with no reads, or `rle` finds no taxon with reads in + every sample (use `tss`); +- the R package MASS cannot be loaded; +- no taxon could be fitted at all. + ### Figures and report The Figures tab lays analysis charts out on a page (A4, letter or a journal @@ -554,13 +647,13 @@ Each entry names a filter in the `filters:` library of `config/composition.yml`. ### Configuring analysis (`analysis:` in `pipeline.yml`) -Controls the defaults applied to the analysis charts (alpha diversity, taxa bar, NMDS, etc.). Per-chart choices such as the taxonomic rank and relative or absolute abundance are made in the UI. +Controls the defaults applied to the analysis charts (alpha diversity, taxa bar, NMDS, differential abundance, etc.). Per-chart choices such as the taxonomic rank and relative or absolute abundance are made in the UI. ```yaml analysis: exclude_categories: # composition categories to drop from figures; [] to keep all - - {set: contamination, category: Contaminant, apply_to: [diversity, taxa, venn]} - # apply_to surfaces: diversity | taxa | composition | venn + - {set: contamination, category: Contaminant, apply_to: [diversity, taxa, venn, differential]} + # apply_to surfaces: diversity | taxa | composition | venn | differential # (omit apply_to to act on every surface) alpha: # richness, Shannon and Simpson normalisation: none # none | rarefy | srs (scaling with ranked subsampling) @@ -577,6 +670,9 @@ analysis: depth: 0 # rarefy/srs reads per sample; 0 = smallest non-empty sample nmds: max_stress: 0.2 # warn if NMDS stress exceeds this value + differential: # negative-binomial differential abundance (MASS::glm.nb) + offset: tss # tss | rle (rle is refused when no taxon has reads in every sample) + min_prevalence: 0.0 # fraction of samples, 0 to 1, a taxon needs reads in to be tested ``` ### Configuring taxonomic filtering (`filters:` in `config/composition.yml`) @@ -762,7 +858,7 @@ The server exposes a REST API under `/api/v1/`. Key endpoint groups: | Results | `GET/POST/DELETE .../results/tables/...` | List, query, filter, save, export (`.xlsx`), and delete tables; OTU member drill-down | | QC | `GET .../results/qc`, `GET .../results/dada2` | MultiQC report metadata and DADA2 figures, logs, stats | | Analysis | `POST .../runs/{run}/analysis/{alpha,chart}`, `GET .../runs/{run}/analysis/{pipeline-stats,ranks}` | Per-run charts and rank discovery | -| Cross-run | `POST /studies/{study}/analysis/{alpha,chart,chart-facet,nmds,permanova,venn,publication-tables}` | Comparison, faceted charts, NMDS, PERMANOVA, taxon overlap and publication tables across runs | +| Cross-run | `POST /studies/{study}/analysis/{alpha,chart,chart-facet,nmds,permanova,venn,publication-tables,differential}` | Comparison, faceted charts, NMDS, PERMANOVA, taxon overlap, publication tables and differential abundance across runs | | Composition | `POST .../runs/{run}/composition/summary`, `POST .../composition/{source}/query`, `POST .../composition/{source}/distinct/{column}` | Organism-composition summaries and tables for a run | | Composition library | `GET /composition`, `POST/DELETE /composition/{filters,sets}/{name}`, `GET /category-sets`, `POST/DELETE /category-sets/{name}` | Edit the filters and category sets in `config/composition.yml` | | Reference trees | `GET/POST /reference-trees`, `GET/PUT/DELETE /reference-trees/{id}`, `GET/PUT .../fasta`, `POST .../run`, `GET .../log/{step}`, `GET .../qc/{align,trim}`, `GET .../alignment/{raw,trimmed}`, `GET .../treefile`, `POST .../trim-preview` | Build and inspect the shared reference trees | diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 8181ae6d..44b2c882 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -358,6 +358,14 @@ analysis: # rle is refused when no taxon is present in every sample. # Used by nb_glm only; clr_welch ignores it, since log-ratios carry no library size. offset: tss + # nb_glm only: where each taxon's negative-binomial theta comes from. ml + # (the default) takes the maximum-likelihood estimate MASS::glm.nb fits per + # taxon. glmGamPoi takes the shrunken dispersion glmGamPoi::glm_gp fits + # across all taxa in one call -- sharing information across taxa, which + # steadies low-count taxa -- and fits each taxon through + # MASS::negative.binomial(theta = 1/dispersion), so the Wald test is + # unchanged. Requires the glmGamPoi R package. + dispersion: ml # Fraction of samples, 0 to 1, in which a taxon must have reads to be # tested. Taxa below it are listed as filtered, without a p-value. For # clr_welch it is applied before zero replacement, so on a sparse table it also diff --git a/docs/occupancy-2026-10.adoc b/docs/occupancy-2026-10.adoc new file mode 100644 index 00000000..6d291b05 --- /dev/null +++ b/docs/occupancy-2026-10.adoc @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) += Occupancy of the vsearch stage +:revdate: 2026-10-14 + +The vsearch stage runs two independent searches, ASV and OTU. The full +pipeline spawns them together; a single-stage re-run runs them one after the +other. The high-water-mark (HWM) algebra of occupancy-types predicts that the +stage's cost is the same in every order but its memory peak is not: serially +the peak is the larger search's, concurrently it can be their sum +(choreographic-types A-2 and A-3). This page records one measurement of that, +on 2026-10-14, and checks it against the typed reading. + +== What was measured + +`MemoryProbe` (`src/core/memory_probe.jl`), off unless a pipeline.yml sets +`occupancy_probe.enabled: true`, records for each attested stage: + +* `julia_allocated_bytes`: the `Base.gc_num()` delta over the stage (cost); +* per task (one spawned search): the largest sampled VmRSS of the child + process it started (`peak_rss_kib`), the kernel's own VmHWM for it + (`hwm_kib`), and the sample window its children were live in — `first_sample`, + the first sample taken after its first child started, and `last_sample`, the + last taken before its last was waited for, 0 for never — which its grade is + read from; +* `child_peak_rss_kib`: the largest sum of the stage's children's VmRSS at + one sampling instant (the stage's occupancy). + +The run: `test/occupancy_disagreement_test.jl`, which runs the same two searches +in three orders under the probe, on a fixture it generates: 1000 reference +sequences of 400 bp, 300 ASV and 150 OTU queries. Linux 6.18 with 4 CPUs, +Julia 1.12.5 with 2 threads, vsearch 2.30.5 with `--threads 1`. Sampling +interval 10 ms. + +== Results + +[cols="2,1,1,1,1,1,1",options="header"] +|=== +| Order | ASV peak (KiB) | OTU peak (KiB) | Stage peak, measured (KiB) | Counted bytes, Session IR | Typed peak, HWM | Julia allocation (bytes) +| ASV then OTU | 9204 | 9180 | 9204 | 9204 | 9204 | 978432 +| OTU then ASV | 9152 | 9192 | 9192 | 9192 | 9192 | 978720 +| Concurrent | 9176 | 9204 | 18380 | 18380 | 18380 | 978816 +|=== + +* *Cost commutes.* The allocation totals agree to 0.04%. The test asserts + agreement within 1%; a planted extra search in one order breaks it (spread + 25%). +* *Each search's peak is the same in every order*, to within 0.6% + (9152-9204 KiB). Each search's VmHWM equals its sampled peak, so the + sampling caught the peaks. +* *The stage peak is not the same.* Serially it is the larger search's + (9192-9204 KiB); concurrently it is 18380 KiB, twice that. This is the A-3 + order dependence, and nothing in the stage's outputs shows it: the hits are + byte-identical in all three orders. The prediction would have been falsified + had the concurrent peak matched the serial one; it did not. +* *Counted bytes equal the typed peak*, in all three orders. The counted bytes + come from occupancy-types' Session IR stepper running + `docs/occupancy/vsearch-*.sir`, one buffer per search. The typed peak is + computed, by `refl`, from occupancy-types' `Timeline` over resource-algebra's + HWM grade (`docs/occupancy/TypedPeak.agda`). A search alone exits holding + nothing (drift 0); in the concurrent order the first is still resident + when the second starts (drift = peak). The sequence law + `max (peak a) (drift a + peak b)` then gives the larger peak serially and the + sum concurrently. +* *The measured stage peak also equals the typed peak* here. For the concurrent + order the typed peak is the interleaving in which both are resident at once; + a measurement could come in lower if the two searches' peaks did not + coincide. Here they did: each search's memory plateaus once its database is + loaded. + +== Reading the numbers + +A recorded `peak_rss_kib` is a reading, not an occupancy: what a stage occupied +depends on what the next stage inherits from it. So the probe grades a stage's +whole window (`MemoryProbe`'s `task_grades`, `stage_grade`, +`predicted_peak_rss_kib`, `cost_bound_rss_kib`, `reclaimed_kib`) and attaches the +result to a recorded stage as `grades`; a record with no sample windows is left +ungraded rather than graded on an assumed order. Tasks are read in sampled order +(first sample, ties by record order). Each handover keeps only the tasks still +live at it, so a task's `drift_kib` is the change in the live peak sum across its +boundary — negative when the boundary also releases an earlier task — and +`release_kib` is everything freed between its start and the next boundary, which +can exceed that task's own peak. The folded drift is the sum of the peaks still +live when the stage ends; the pinned instance's own `drift` is that non-negative +live sum, the signed per-task delta being the extension its header records as +H-2. + +For the three orders above, the algebra's predicted peak reproduces the measured +stage peak exactly — 9204, 9192 and 18380 KiB — and the slack against the cost +bound is what the cuts reclaimed: 9180, 9152 and 0 KiB. +`scripts/check-occupancy-grades.jl` checks that equality, occupancy≤cost and +non-negative reclamation against this page's raw record, rebuilding each order's +windows from `docs/occupancy/schedules.yml` (the file records no windows of its +own; the Session IR texts in this directory state the same interleavings). The +reclamation readings are theorems over the pinned siblings at these numbers: +`docs/occupancy/GradedStages.agda` — `no-reclaim` for the concurrent stage, +`reclaimed` for the serial one, where 9204 KiB are released at the frontier and +the cost reading counts them — and the two protocol-level statements +(`occupancy-at-cut`, `protocol-occupancy≤cost`) as the siblings' own two-step +over the stage's timeline. `docs/occupancy/reject` gains a fourth rejection, for +reading the concurrent stage's drift as 0. + +`scripts/lint-occupancy-readings.sh` keeps production code from reading the key +directly: outside `src/core/memory_probe.jl`, an occupancy figure must go through +those helpers. + +== Status + +[cols="1,3,1,2",options="header"] +|=== +| ID | Claim | Status | Artefact +| MMW-01.1 | With the flag off, a stage record has exactly its earlier keys; with it on, it gains `occupancy`; old records still parse and merge | TESTED | `test/occupancy_disagreement_test.jl` ("flag off", "round trip", "through the pipeline's _attest") +| MMW-01.2 | For this workload, the vsearch stage's cost (Julia allocation) is the same in all three orders | TESTED | same file, "two-thread disagreement" (asserted, within 1%) +| MMW-01.3 | For this workload, the stage peak depends on the order (A-3), and each search's own peak does not | TESTED | same file, recorded in the test log; table above +| MMW-01.4 | For the three Session IR texts, the stepper's counted bytes equal the HWM typed peak | TESTED | `scripts/check-occupancy.sh` (stepper output against `TypedPeak.agda`; three rejections in `docs/occupancy/reject`) +| MMW-01.5 | The recorded run's grades reproduce its measured stage peaks, and the reclamation the pinned siblings' laws read off them is what the cuts freed | TESTED | `scripts/check-occupancy-grades.jl` (the recorded record), `docs/occupancy/GradedStages.agda` (`no-reclaim`, `reclaimed`, `occupancy-at-cut` at the measured numbers), `test/unit/test_occupancy_grade.jl` +| bridge | Counted bytes are bounded by the typed peak in general (occupancy-types' v0 → typed bridge) | OPEN | unchanged; this page gives three instances, not a proof +|=== + +== Limits + +* *Sampled.* A per-task peak is the largest VmRSS seen at 10 ms intervals. VmHWM + is exact up to the last sample, but growth after it is not seen. A child that + lives for less than one interval may not be sampled at all (`samples: 0`). +* *Child memory only.* The children are counted; the Julia process is not. Its + `julia_live_bytes_delta` is recorded, but without a forced collection it + reflects when the GC last ran, not what the stage kept. +* *Allocation is process-wide.* While the study loop runs several runs' + pipelines at once, a stage's `julia_allocated_bytes` includes the others' + allocations too. The child figures are per stage, since each stage samples + only the children it started. +* *Linux only*, since it reads /proc. For a stage offloaded with `remote:`, + the local child is an ssh client, so its memory says nothing about the work. +* *One thread for vsearch.* With several threads, vsearch writes its hits in + whatever order the threads finish. `_sort_hits_by_query!` then does a + different amount of work, because its sort key splits each line on every + comparison. That moved the stage's allocation total by up to 12% between + repeats of the same order. So in production the cost is order-independent + only up to that noise; precomputing the sort key would remove it. This PR + does not change it. +* *One host, one day.* CI runs the same test on every push and logs its own + numbers; they are not compared with these. + +== Reproducing + +---- +julia --project=. -t 2 test/occupancy_disagreement_test.jl # set OCCUPANCY_RESULTS= for YAML +bash scripts/check-occupancy.sh # needs julia and agda 2.6.4.x +julia --project=. scripts/check-occupancy-grades.jl # the recorded run's grades +bash scripts/lint-occupancy-readings.sh # no ungraded readings in src/ +---- + +`scripts/fetch-siblings.sh` fetches the siblings at their pins into +`_build/dependencies`: occupancy-types `af95728` and resource-algebra +`cf9235c`. + +== As the proofs progress + +* *Per-buffer (n-ary) occupancy* in occupancy-types: each search is already one + buffer in the Session IR texts, so the typed peak could then be read per + search. +* *Signed drift*: frees could be graded as negative drift, so the Julia side's + collections (the pipeline's `GC.gc(false)` hints) become modelled + reclamation rather than unmodelled noise. +* *choreographic-types' operational `step` as a function*: the three orders + become three traces of one typed protocol. The frontier certificate its + two-thread application note describes could then be displayed in the WebUI. diff --git a/docs/occupancy/GradedStages.agda b/docs/occupancy/GradedStages.agda new file mode 100644 index 00000000..3c45730d --- /dev/null +++ b/docs/occupancy/GradedStages.agda @@ -0,0 +1,96 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- The vsearch stage's grades, and occupancy-types' reclamation theorems at the +-- measured numbers. Fork issue #105 asks for exactly this: grade the stages +-- (alloc/hold/release) and compare the predicted peak with the measured one. +-- TypedPeak.agda computes the typed peak by the algebra's sequence law; this +-- module adds the grades the measurement itself gives, and the laws that say +-- what the difference between a peak and a cost reading is: occupancy≤cost, and +-- the strict reclamation at a cut where something is released. +-- +-- Consumed, never re-declared, from the siblings scripts/fetch-siblings.sh pins +-- and scripts/check-occupancy.sh checks: +-- occupancy-types af957280075699c659de0abd9a3b0defade77309 +-- resource-algebra cf9235c532712a783a404c359a1055f40134794a +-- Everything is stated in this file's own Timeline instance (`TH`, as +-- TypedPeak.agda does): the Grade-level laws are Reclaim's as they stand, and +-- the two protocol-level ones are Reclaim's own two-step — its `cut-factor` over +-- its `reclaim-law`/`occupancy≤cost` — applied to `xs` and `ys` of that instance. +-- +-- A grade is `hwm peak drift _`: what an interaction reached, and what it still +-- holds when the next one begins. The peaks below are the 2026-10-14 run's +-- (docs/occupancy-2026-10.adoc, docs/occupancy/results-2026-10-14.yml), in KiB. +module GradedStages where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import OccupancyTypes.Order +open import OccupancyTypes.Reclaim + +import OccupancyTypes.Timeline +module TH = OccupancyTypes.Timeline HWM + +-- A search that has not been waited for when the next one starts: it still holds +-- everything it reached, so the cut reclaims none of it. +still : Nat → Grade +still p = hwm p p (≤-refl p) + +-- A search that has exited before the next one starts: it holds nothing, so the +-- cut reclaims all of it. +gone : Nat → Grade +gone p = hwm p 0 z≤n + +-- The full pipeline runs the two searches together: ASV is still resident when +-- OTU starts, so the stage peak is their sum and nothing is reclaimed — +-- Reclaim.no-reclaim's equality, whose `refl` argument discharges the theorem's +-- own hypothesis (drift = peak) at the measured grade. +run-together : peak (still 9176 ⊗h gone 9204) ≡ 18380 +run-together = no-reclaim (still 9176) refl (gone 9204) + +-- A single-stage re-run runs them one after the other: ASV has exited, so the +-- cut reclaims its 9204 KiB and the stage peak is the larger search's +-- (Reclaim.reclaim-law, by `refl`). +run-in-turn : peak (gone 9204 ⊗h gone 9180) ≡ 9204 +run-in-turn = refl + +-- The same claim as Reclaim.reclaimed states it: with drift < peak at the cut +-- and a suffix that does anything, occupancy is strictly below cost. 9204 KiB +-- are released at the frontier, so the cost reading (18384 KiB) overstates the +-- stage. +run-in-turn +-- +-- The vsearch stage's typed peak, computed (TESTED, by refl) with +-- occupancy-types' Timeline over resource-algebra's HWM grade, both consumed +-- by pin through scripts/fetch-siblings.sh: +-- occupancy-types af957280075699c659de0abd9a3b0defade77309 +-- resource-algebra cf9235c532712a783a404c359a1055f40134794a +-- Nothing here re-declares their definitions. +-- +-- Each search is one interaction graded by its measured peak, in KiB, from the +-- 2026-10-14 run in docs/occupancy-2026-10.adoc. A search's process exits +-- holding nothing, so run alone its drift is 0. Run concurrently, the first is +-- still resident when the second starts, so that is graded with drift = peak. +-- That is the interleaving the measurement observed (both resident at once). +-- The algebra's sequence law, max (peak a) (drift a + peak b), then gives: +-- serial = the larger peak; concurrent = the sum. +module TypedPeak where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM + +import OccupancyTypes.Timeline +module TH = OccupancyTypes.Timeline HWM + +-- A search whose process has exited before the next one starts. +exits : Nat → Grade +exits p = hwm p 0 z≤n + +-- A search still resident when the next one starts. +resident : Nat → Grade +resident p = hwm p p (≤-refl p) + +-- ASV 9204 then OTU 9180. +asv-then-otu : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 9204 +asv-then-otu = refl + +-- OTU 9192 then ASV 9152. +otu-then-asv : peak (TH.grade (exits 9192 TH.∷ exits 9152 TH.∷ TH.[])) ≡ 9192 +otu-then-asv = refl + +-- ASV 9176 and OTU 9204 together. +concurrent : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 18380 +concurrent = refl diff --git a/docs/occupancy/reject/ConcurrentDriftIsZero.agda b/docs/occupancy/reject/ConcurrentDriftIsZero.agda new file mode 100644 index 00000000..749c213f --- /dev/null +++ b/docs/occupancy/reject/ConcurrentDriftIsZero.agda @@ -0,0 +1,18 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: reading the concurrent stage as one that hands on nothing. +-- Running the two searches together leaves the ASV search resident when the OTU +-- search starts, so the stage's drift is that search's peak — 9176 KiB, what a +-- suffix would see — and not 0. Grading the drift as 0 would read the stage's +-- COST as its occupancy, which is the very reading the measured overlap exists +-- to refute. +module ConcurrentDriftIsZero where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import GradedStages + +wrong : drift (still 9176 ⊗h gone 9204) ≡ 0 +wrong = refl diff --git a/docs/occupancy/reject/ConcurrentIsSerial.agda b/docs/occupancy/reject/ConcurrentIsSerial.agda new file mode 100644 index 00000000..592de6ee --- /dev/null +++ b/docs/occupancy/reject/ConcurrentIsSerial.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: the claim that running the two searches concurrently leaves +-- the peak where a serial run puts it (the larger of the two). +module ConcurrentIsSerial where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 9204 +wrong = refl diff --git a/docs/occupancy/reject/SerialIsCost.agda b/docs/occupancy/reject/SerialIsCost.agda new file mode 100644 index 00000000..dbb6f4ba --- /dev/null +++ b/docs/occupancy/reject/SerialIsCost.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: reading a serial run's peak as the sum of the two peaks, +-- i.e. occupancy as cost (occupancy-types R-6). +module SerialIsCost where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 18384 +wrong = refl diff --git a/docs/occupancy/results-2026-10-14.yml b/docs/occupancy/results-2026-10-14.yml new file mode 100644 index 00000000..394236d6 --- /dev/null +++ b/docs/occupancy/results-2026-10-14.yml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# The raw MemoryProbe records behind docs/occupancy-2026-10.adoc, written by +# test/occupancy_disagreement_test.jl with OCCUPANCY_RESULTS set (2026-10-14). +vsearch: "vsearch v2.30.5_linux_x86_64, 15.7GB RAM, 4 cores" +julia: "1.12.5" +threads: 2 +cpus: 4 +cost_spread: 0.00039231071008238524 +orders: + asv_then_otu: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 56 + julia_allocated_bytes: 978432 + julia_live_bytes_delta: 978544 + child_peak_rss_kib: 9204 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9204 + hwm_kib: 9204 + - label: "vsearch:otu" + processes: 1 + samples: 20 + peak_rss_kib: 9180 + hwm_kib: 9180 + otu_then_asv: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 57 + julia_allocated_bytes: 978720 + julia_live_bytes_delta: 978832 + child_peak_rss_kib: 9192 + tasks: + - label: "vsearch:otu" + processes: 1 + samples: 22 + peak_rss_kib: 9192 + hwm_kib: 9192 + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9152 + hwm_kib: 9152 + concurrent: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 33 + julia_allocated_bytes: 978816 + julia_live_bytes_delta: 978928 + child_peak_rss_kib: 18380 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 30 + peak_rss_kib: 9176 + hwm_kib: 9176 + - label: "vsearch:otu" + processes: 1 + samples: 24 + peak_rss_kib: 9204 + hwm_kib: 9204 diff --git a/docs/occupancy/schedules.yml b/docs/occupancy/schedules.yml new file mode 100644 index 00000000..dfdedb81 --- /dev/null +++ b/docs/occupancy/schedules.yml @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# The interleaving each order in results-2026-10-14.yml ran with, as the sample +# window [first, last] of each of its tasks, in the task order the record keeps. +# These are the same schedules the Session IR texts in this directory state +# (vsearch-concurrent.sir runs the two searches together; the other two run them +# one after the other), expressed in the probe's own sample indices: the windows +# of a serial order do not overlap, and the concurrent order's do. +# +# Read by test/unit/test_occupancy_grade.jl and scripts/check-occupancy-grades.jl, +# so the two cannot disagree about what the recorded run did. +asv_then_otu: + - [1, 2] + - [3, 4] +otu_then_asv: + - [1, 2] + - [3, 4] +concurrent: + - [1, 4] + - [2, 5] diff --git a/docs/occupancy/vsearch-asv-then-otu.sir b/docs/occupancy/vsearch-asv-then-otu.sir new file mode 100644 index 00000000..e462fb5e --- /dev/null +++ b/docs/occupancy/vsearch-asv-then-otu.sir @@ -0,0 +1,13 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-asv-then-otu.sir - the vsearch stage with the two searches run one + after the other, ASV first: the single-stage re-run's order. Each search is + one buffer, sized by its measured peak in KiB (docs/occupancy-2026-10.adoc, + 2026-10-14 run); each process exits holding nothing, so its buffer is dropped + before the next search starts. + + Expected: steps = 0 (no communication), peak_live = max 9204 9180 = 9204. *) +let asv = alloc 9204 in +let _ = drop asv in +let otu = alloc 9180 in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-concurrent.sir b/docs/occupancy/vsearch-concurrent.sir new file mode 100644 index 00000000..93be40bf --- /dev/null +++ b/docs/occupancy/vsearch-concurrent.sir @@ -0,0 +1,12 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-concurrent.sir - the two searches spawned together, as the full + pipeline runs them. Session IR has no parallel composition, so this is the + interleaving the measurement observed: both buffers live at once, then both + dropped. + + Expected: steps = 0, peak_live = 9176 + 9204 = 18380. *) +let asv = alloc 9176 in +let otu = alloc 9204 in +let _ = drop asv in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-otu-then-asv.sir b/docs/occupancy/vsearch-otu-then-asv.sir new file mode 100644 index 00000000..ba575591 --- /dev/null +++ b/docs/occupancy/vsearch-otu-then-asv.sir @@ -0,0 +1,9 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-otu-then-asv.sir - the same two searches, OTU first. + + Expected: steps = 0, peak_live = max 9192 9152 = 9192. *) +let otu = alloc 9192 in +let _ = drop otu in +let asv = alloc 9152 in +let _ = drop asv in +unit diff --git a/frontend/bun.lock b/frontend/bun.lock index 742f42a5..11ec980d 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -7,226 +7,128 @@ "dependencies": { "@upsetjs/react": "^1.11.0", "jspdf": "^4.2.1", - "plotly.js-dist-min": "^2.35.2", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "react-router-dom": "^6.28.0", + "plotly.js-dist-min": "^4.1.2", + "react": "^19.3.0", + "react-dom": "^19.3.0", + "react-router-dom": "^7.18.4", "svg2pdf.js": "^2.8.1", }, "devDependencies": { - "@types/plotly.js": "^2.33.4", - "@types/react": "^18.3.18", - "@types/react-dom": "^18.3.5", - "@vitejs/plugin-react": "^4.3.4", - "typescript": "^5.7.2", - "vite": "^6.1.0", + "@types/plotly.js": "^3.0.15", + "@types/react": "^19.3.0", + "@types/react-dom": "^19.3.0", + "@vitejs/plugin-react": "^6.1.2", + "typescript": "^7.0.2", + "vite": "^8.3.4", }, }, }, "overrides": { - "@types/react": "^18.3.18", - "@types/react-dom": "^18.3.5", + "@types/react": "^19.3.0", + "@types/react-dom": "^19.3.0", }, "packages": { - "@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], - - "@babel/compat-data": ["@babel/compat-data@7.29.0", "", {}, "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg=="], - - "@babel/core": ["@babel/core@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-compilation-targets": "^7.28.6", "@babel/helper-module-transforms": "^7.28.6", "@babel/helpers": "^7.28.6", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/traverse": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA=="], - - "@babel/generator": ["@babel/generator@7.29.1", "", { "dependencies": { "@babel/parser": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw=="], - - "@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.28.6", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA=="], - - "@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="], - - "@babel/helper-module-imports": ["@babel/helper-module-imports@7.28.6", "", { "dependencies": { "@babel/traverse": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw=="], - - "@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.6", "", { "dependencies": { "@babel/helper-module-imports": "^7.28.6", "@babel/helper-validator-identifier": "^7.28.5", "@babel/traverse": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA=="], - - "@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.28.6", "", {}, "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug=="], - - "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], - - "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], - - "@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="], - - "@babel/helpers": ["@babel/helpers@7.28.6", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw=="], - - "@babel/parser": ["@babel/parser@7.29.0", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww=="], - - "@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="], - - "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="], - "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], - "@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="], - - "@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="], - - "@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], - - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], - - "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], - - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], - - "@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], - - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], - - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], - - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], - - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], - - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], - - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], - - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], - - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], - - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], - - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], - - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], - - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], - - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], - - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], - - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + "@oxc-project/types": ["@oxc-project/types@0.154.0", "", {}, "sha512-KYOUGQFlLWR+s4QMV2Q9ko38TZm7EdO87Qm7rvZLqk/webse/cDjtyxKhg/47l0byCbs/k46LT8qc4PKjAikzA=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.14", "", { "os": "android", "cpu": "arm" }, "sha512-aWR+XKH9+rW7m5fWq/5360J3mHycrx/o6xZM2DQFdJnTMdyP7/WzxPjhtLbVWdKo2y612/4U1bWqb3HQhiF7vg=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.2.14", "", { "os": "android", "cpu": "arm64" }, "sha512-QiVxHofl6A8lQU7WAtOUY4QMVZXohxwz7WJJoeh8NBEUyKfIkMRKhYyq8az/1RypNiIXzNTtiS6KwHz0erxYRQ=="], - "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="], + "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.2.14", "", { "os": "darwin", "cpu": "arm64" }, "sha512-lTQOzHuBIooh3jIr5XINQkjPGEuGy+Dsy0meXDr6Sjty0b99kYgGh4uBC/8mJX1T0kC8d6S0aPP1Y+bHkh6aBw=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.2.14", "", { "os": "darwin", "cpu": "x64" }, "sha512-hyMKEGatjmBYCs/Q4sbIcL/GqpFEqsX3wc+HYVQ/85A4rxc1o7rPm4DUI4GV4k/h0ITddwFSGpqQunI14GXjTA=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.2.14", "", { "os": "freebsd", "cpu": "x64" }, "sha512-u371GbXoVXrEgP20PwzoGNOdAOqAMhzo19JzzY+V4YmmnC+DTOoqdbMu+O+54emM59Mik/UdrtWgcuEDKWRPKA=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.2.14", "", { "os": "linux", "cpu": "arm" }, "sha512-BqwKsdZQf6EqVWr8fv2MrDYzAvOZ7+NYFSJESRGs87fkseEhs0TU9TXgRJTkZoRRkxTizyEg1LKUXfU3/78uqg=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.2.14", "", { "os": "linux", "cpu": "arm64" }, "sha512-SD9kY8M009x/7s57Z1dgUVlP5+uYBgKQK0Tkr1WIWngrtUJmRCW1RJFMY7QDla2khXhmg2i5Io7MUjcmZNZ/KQ=="], - "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], + "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.2.14", "", { "os": "linux", "cpu": "arm64" }, "sha512-BJ2CL19M6uuWesMgnqJzgxoI+N+eV7b0A9O9vLV42y+gitC41fNa8KElau1tHVzuLWX5GzEuxMPhAO0tQXQMkg=="], - "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], + "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.2.14", "", { "os": "linux", "cpu": "ppc64" }, "sha512-4+Kl8daPV8NDHaD7mfCcc1pRM+Qg7ab8DyMm5FG5WoZgv4h2i3KFN1CybYBNbQqmuYm7ftJLvmMI1eHYJ22S/Q=="], - "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.2.14", "", { "os": "linux", "cpu": "s390x" }, "sha512-V/h8ePq1fKDrmtFMz6xcCN3DKvuvMRrwwD1q0PkBeeo4DpK64biaKvqWGVxqCeYKqeRk7gE7/iKPxLCn08O97A=="], - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.2.14", "", { "os": "linux", "cpu": "x64" }, "sha512-WxalXzgrLVpKC93xfpaJ3gCXUWzeCr+FRCbDVD2mUAy4L6Vo3lNVz0epUEUpxPWJkLNYUwM4OSIikv36OJ9mLg=="], - "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.2.14", "", { "os": "linux", "cpu": "x64" }, "sha512-3G3tnAmTpdk9jO3GQtyHXqeFD2SlecfTws/Y3M2mmj9nMBwwuLT4kur8sZKeu7ENUHSpVO3cIsm5A52LEWWwlg=="], - "@remix-run/router": ["@remix-run/router@1.23.2", "", {}, "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w=="], + "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.2.14", "", { "os": "none", "cpu": "arm64" }, "sha512-qyr5XiNGxm2ytx17vvtcJ9nXn6v1gENETi503bJx3OGwZ1tSonj7BosxbMa/xeAfSl5iS9X6KjEWYXLu6v1L7Q=="], - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="], + "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.2.14", "", { "os": "win32", "cpu": "arm64" }, "sha512-oA4o5o0S2yB199YyYXDzKUv1Z56Rc+mgOjwm7Z/vJ9Z8igiDPYvoOuDyo6g7AK/ZaF09nPhDKVnGBcYHO11GXA=="], - "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.59.0", "", { "os": "android", "cpu": "arm" }, "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg=="], + "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.2.14", "", { "os": "win32", "cpu": "x64" }, "sha512-9mE94bf2GZUaR/fUSXCMpBoWkctu421Mi0ORpm4zqfbkjKep7RxzPFa5etcGs5h+2KApoWBOA/hAJa/7saGgEQ=="], - "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.59.0", "", { "os": "android", "cpu": "arm64" }, "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], - "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.59.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg=="], - - "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.59.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w=="], - - "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.59.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA=="], - - "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.59.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg=="], - - "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw=="], - - "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA=="], - - "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA=="], - - "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA=="], - - "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg=="], - - "@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q=="], + "@types/lz-string": ["@types/lz-string@1.5.0", "", { "dependencies": { "lz-string": "*" } }, "sha512-s84fKOrzqqNCAPljhVyC5TjAo6BH4jKHw9NRNFNiRUY5QSgZCmVm5XILlWbisiKl+0OcS7eWihmKGS5akc2iQw=="], - "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA=="], + "@types/pako": ["@types/pako@2.0.4", "", {}, "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw=="], - "@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA=="], + "@types/plotly.js": ["@types/plotly.js@3.0.15", "", {}, "sha512-ftBxk2fB7BxxeMpT0IVEb/3HKpYTXO5nCdR5mGEz6gHpLd6GGRFxvDVoG1gpD/gBAgoN36KIyRflxCnOfWlskQ=="], - "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg=="], + "@types/raf": ["@types/raf@3.4.3", "", {}, "sha512-c4YAvMedbPZ5tEyxzQdMoOhhJ4RD3rngZIdwC2/qDN3d7JpEhB6fiBRKVY1lg5B7Wk+uPBjn5f39j1/2MY1oOw=="], - "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg=="], + "@types/react": ["@types/react@19.3.0", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg=="], - "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.59.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w=="], + "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], - "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg=="], + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], - "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg=="], + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], - "@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.59.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ=="], + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], - "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.59.0", "", { "os": "none", "cpu": "arm64" }, "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA=="], + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], - "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.59.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A=="], + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], - "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.59.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA=="], + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], - "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA=="], + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], - "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA=="], + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], - "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], - "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], - "@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="], + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], - "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], - "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], - "@types/lz-string": ["@types/lz-string@1.5.0", "", { "dependencies": { "lz-string": "*" } }, "sha512-s84fKOrzqqNCAPljhVyC5TjAo6BH4jKHw9NRNFNiRUY5QSgZCmVm5XILlWbisiKl+0OcS7eWihmKGS5akc2iQw=="], + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], - "@types/pako": ["@types/pako@2.0.4", "", {}, "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw=="], + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], - "@types/plotly.js": ["@types/plotly.js@2.35.14", "", {}, "sha512-CcD/32JcK19+xWH4FFpmYez/5X9kOjUcBr8Hxh7gQ/3Z32gIoLLy/L9xvC7DG5YikPvJjq6QN05B9+MCRu/Ncw=="], + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], - "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], - "@types/raf": ["@types/raf@3.4.3", "", {}, "sha512-c4YAvMedbPZ5tEyxzQdMoOhhJ4RD3rngZIdwC2/qDN3d7JpEhB6fiBRKVY1lg5B7Wk+uPBjn5f39j1/2MY1oOw=="], + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], - "@types/react": ["@types/react@18.3.28", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw=="], + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], - "@types/react-dom": ["@types/react-dom@18.3.7", "", { "peerDependencies": { "@types/react": "^18.0.0" } }, "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ=="], + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], - "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], "@upsetjs/model": ["@upsetjs/model@1.11.0", "", {}, "sha512-pQEdunei2WC9mdhFR6umzE45j8VJfei/cG0TyVaaC0YE5zh1WI/TJRJJQC5jCiGhylISy99ANJs6XmQI33lqCw=="], "@upsetjs/react": ["@upsetjs/react@1.11.0", "", { "dependencies": { "@types/lz-string": "^1.3.34", "@types/react": "^17.0.38", "@upsetjs/model": "~1.11.0", "lz-string": "^1.4.4" }, "peerDependencies": { "react": ">=17" } }, "sha512-APDFkXowPk1U94Iz5dtzpjcSiR5GcJkhLdMdeTnyLDYxIBf1mA+ElbBrEdAzaSGCiZo76ZKfgmuCMSMEfLtN8g=="], - "@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@6.1.2", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "oxc-transform-react": "^0.152.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler", "oxc-transform-react"] }, "sha512-fhJNGUdscQa4HkI0xFxwonWyhnVReCPDbXI7FH7jmzpnVCrFQI7PRrhWecO263610vOrs9tr2Y4xcf8uAdFxbw=="], "base64-arraybuffer": ["base64-arraybuffer@1.0.2", "", {}, "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ=="], - "baseline-browser-mapping": ["baseline-browser-mapping@2.10.8", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-PCLz/LXGBsNTErbtB6i5u4eLpHeMfi93aUv5duMmj6caNu6IphS4q6UevDnL36sZQv9lrP11dbPKGMaXPwMKfQ=="], - - "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], - - "caniuse-lite": ["caniuse-lite@1.0.30001778", "", {}, "sha512-PN7uxFL+ExFJO61aVmP1aIEG4i9whQd4eoSCebav62UwDyp5OHh06zN4jqKSMePVgxHifCw1QJxdRkA1Pisekg=="], - "canvg": ["canvg@3.0.11", "", { "dependencies": { "@babel/runtime": "^7.12.5", "@types/raf": "^3.4.0", "core-js": "^3.8.3", "raf": "^3.4.1", "regenerator-runtime": "^0.13.7", "rgbcolor": "^1.0.1", "stackblur-canvas": "^2.0.0", "svg-pathdata": "^6.0.3" } }, "sha512-5ON+q7jCTgMp9cjpu4Jo6XbvfYwSB2Ow3kzHKfIyJfaCAOHLbdKPQqGKgfED/R5B+3TFFfe8pegYA+b423SRyA=="], - "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], + "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], "core-js": ["core-js@3.49.0", "", {}, "sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg=="], @@ -236,16 +138,10 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], - "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "dompurify": ["dompurify@3.4.16", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ=="], - "electron-to-chromium": ["electron-to-chromium@1.5.313", "", {}, "sha512-QBMrTWEf00GXZmJyx2lbYD45jpI3TUFnNIzJ5BBc8piGUDwMPa1GV6HJWTZVvY/eiN3fSopl7NRbgGp9sZ9LTA=="], - - "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], - - "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], - "fast-png": ["fast-png@6.4.0", "", { "dependencies": { "@types/pako": "^2.0.3", "iobuffer": "^5.3.2", "pako": "^2.1.0" } }, "sha512-kAqZq1TlgBjZcLr5mcN6NP5Rv4V2f22z00c3g8vRrwkcqjerx7BEhPbOnWCPqaHUl2XWQBJQvOT/FQhdMT7X/Q=="], "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], @@ -256,31 +152,39 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], - "html2canvas": ["html2canvas@1.4.1", "", { "dependencies": { "css-line-break": "^2.1.0", "text-segmentation": "^1.0.3" } }, "sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA=="], "iobuffer": ["iobuffer@5.4.0", "", {}, "sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA=="], - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "jspdf": ["jspdf@4.2.1", "", { "dependencies": { "@babel/runtime": "^7.28.6", "fast-png": "^6.2.0", "fflate": "^0.8.1" }, "optionalDependencies": { "canvg": "^3.0.11", "core-js": "^3.6.0", "dompurify": "^3.3.1", "html2canvas": "^1.0.0-rc.5" } }, "sha512-YyAXyvnmjTbR4bHQRLzex3CuINCDlQnBqoSYyjJwTP2x9jDLuKDzy7aKUl0hgx3uhcl7xzg32agn5vlie6HIlQ=="], - "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], + "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], - "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "lightningcss-android-arm64": ["lightningcss-android-arm64@1.33.0", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="], - "jspdf": ["jspdf@4.2.1", "", { "dependencies": { "@babel/runtime": "^7.28.6", "fast-png": "^6.2.0", "fflate": "^0.8.1" }, "optionalDependencies": { "canvg": "^3.0.11", "core-js": "^3.6.0", "dompurify": "^3.3.1", "html2canvas": "^1.0.0-rc.5" } }, "sha512-YyAXyvnmjTbR4bHQRLzex3CuINCDlQnBqoSYyjJwTP2x9jDLuKDzy7aKUl0hgx3uhcl7xzg32agn5vlie6HIlQ=="], + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.33.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg=="], - "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.33.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ=="], - "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.33.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg=="], - "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.33.0", "", { "os": "linux", "cpu": "arm" }, "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ=="], - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg=="], - "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ=="], - "node-releases": ["node-releases@2.0.36", "", {}, "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA=="], + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg=="], + + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw=="], + + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.33.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA=="], + + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], + + "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], + + "nanoid": ["nanoid@3.3.20", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ=="], "pako": ["pako@2.2.0", "", {}, "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w=="], @@ -288,35 +192,33 @@ "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "picomatch": ["picomatch@4.0.7", "", {}, "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA=="], - "plotly.js-dist-min": ["plotly.js-dist-min@2.35.3", "", {}, "sha512-sz2HLP8gkysLx/BanM2PtJTtZ1PLPwdHwMWNri2YxLBy3IOeuDsVQtlmWa4hoK3j/fi4naaD3uZJqH5ozM3zGg=="], + "plotly.js-dist-min": ["plotly.js-dist-min@4.1.2", "", {}, "sha512-VSmS9KVNun16S8qFYYp9LlL/XBby4ueeGqB+K9AqRCVsup8dSpGULap4UyR1f2mz4pV3qjWp1sR5chwLgIKPoQ=="], - "postcss": ["postcss@8.5.8", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg=="], + "postcss": ["postcss@8.5.30", "", { "dependencies": { "nanoid": "^3.3.20", "picocolors": "^1.1.1", "source-map-js": "^1.2.2" } }, "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA=="], "raf": ["raf@3.4.1", "", { "dependencies": { "performance-now": "^2.1.0" } }, "sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA=="], - "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], - - "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], + "react": ["react@19.3.0", "", {}, "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog=="], - "react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="], + "react-dom": ["react-dom@19.3.0", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.0" } }, "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q=="], - "react-router": ["react-router@6.30.3", "", { "dependencies": { "@remix-run/router": "1.23.2" }, "peerDependencies": { "react": ">=16.8" } }, "sha512-XRnlbKMTmktBkjCLE8/XcZFlnHvr2Ltdr1eJX4idL55/9BbORzyZEaIkBFDhFGCEWBBItsVrDxwx3gnisMitdw=="], + "react-router": ["react-router@7.18.4", "", { "dependencies": { "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, "optionalPeers": ["react-dom"] }, "sha512-PUPQcMhMGRAslLcvtlPz/kmzBEWPhLdgLFrL7pLNepBL6dX0lWj4WD2cUYVgYCuT3jxvghYFg81cDTj44DhetQ=="], - "react-router-dom": ["react-router-dom@6.30.3", "", { "dependencies": { "@remix-run/router": "1.23.2", "react-router": "6.30.3" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-pxPcv1AczD4vso7G4Z3TKcvlxK7g7TNt3/FNGMhfqyntocvYKj+GCatfigGDjbLozC4baguJ0ReCigoDJXb0ag=="], + "react-router-dom": ["react-router-dom@7.18.4", "", { "dependencies": { "react-router": "7.18.4" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-yrfmJHIpDG7taCpqKjT1G5B6q3O2K+RN8/fgNf0lTjCwiPbQ0ei6vXX9ZjQR+7ld8Tr7Z5xmyMnZ8YJrphWQUw=="], "regenerator-runtime": ["regenerator-runtime@0.13.11", "", {}, "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="], "rgbcolor": ["rgbcolor@1.0.1", "", {}, "sha512-9aZLIrhRaD97sgVhtJOW6ckOEh6/GnvQtdVNfdZ6s67+3/XwLS9lBcQYzEEhYVeUowN7pRzMLsyGhK2i/xvWbw=="], - "rollup": ["rollup@4.59.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.59.0", "@rollup/rollup-android-arm64": "4.59.0", "@rollup/rollup-darwin-arm64": "4.59.0", "@rollup/rollup-darwin-x64": "4.59.0", "@rollup/rollup-freebsd-arm64": "4.59.0", "@rollup/rollup-freebsd-x64": "4.59.0", "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", "@rollup/rollup-linux-arm-musleabihf": "4.59.0", "@rollup/rollup-linux-arm64-gnu": "4.59.0", "@rollup/rollup-linux-arm64-musl": "4.59.0", "@rollup/rollup-linux-loong64-gnu": "4.59.0", "@rollup/rollup-linux-loong64-musl": "4.59.0", "@rollup/rollup-linux-ppc64-gnu": "4.59.0", "@rollup/rollup-linux-ppc64-musl": "4.59.0", "@rollup/rollup-linux-riscv64-gnu": "4.59.0", "@rollup/rollup-linux-riscv64-musl": "4.59.0", "@rollup/rollup-linux-s390x-gnu": "4.59.0", "@rollup/rollup-linux-x64-gnu": "4.59.0", "@rollup/rollup-linux-x64-musl": "4.59.0", "@rollup/rollup-openbsd-x64": "4.59.0", "@rollup/rollup-openharmony-arm64": "4.59.0", "@rollup/rollup-win32-arm64-msvc": "4.59.0", "@rollup/rollup-win32-ia32-msvc": "4.59.0", "@rollup/rollup-win32-x64-gnu": "4.59.0", "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg=="], + "rolldown": ["rolldown@1.2.14", "", { "dependencies": { "@oxc-project/types": "=0.154.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.2.14", "@rolldown/binding-android-arm64": "1.2.14", "@rolldown/binding-darwin-arm64": "1.2.14", "@rolldown/binding-darwin-x64": "1.2.14", "@rolldown/binding-freebsd-x64": "1.2.14", "@rolldown/binding-linux-arm-gnueabihf": "1.2.14", "@rolldown/binding-linux-arm64-gnu": "1.2.14", "@rolldown/binding-linux-arm64-musl": "1.2.14", "@rolldown/binding-linux-ppc64-gnu": "1.2.14", "@rolldown/binding-linux-s390x-gnu": "1.2.14", "@rolldown/binding-linux-x64-gnu": "1.2.14", "@rolldown/binding-linux-x64-musl": "1.2.14", "@rolldown/binding-openharmony-arm64": "1.2.14", "@rolldown/binding-win32-arm64-msvc": "1.2.14", "@rolldown/binding-win32-x64-msvc": "1.2.14" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-eTBTTqMR6uHKLgcUqpJP8ntzDI6V4LYA8negAMBEI7gkgVvOdrx6VsvS2Ym+83KhOPdLCHWJu0w7UQFJZhHKNw=="], - "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], + "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], - "semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "set-cookie-parser": ["set-cookie-parser@2.7.2", "", {}, "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw=="], - "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], "specificity": ["specificity@0.4.1", "", { "bin": { "specificity": "./bin/specificity" } }, "sha512-1klA3Gi5PD1Wv9Q0wUoOQN1IWAuPu0D1U03ThXTr0cJ20+/iq2tHSDnK7Kk/0LXJ1ztUB2/1Os0wKmfyNgUQfg=="], @@ -330,16 +232,12 @@ "text-segmentation": ["text-segmentation@1.0.3", "", { "dependencies": { "utrie": "^1.0.2" } }, "sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw=="], - "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], + "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "utrie": ["utrie@1.0.2", "", { "dependencies": { "base64-arraybuffer": "^1.0.2" } }, "sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw=="], - "vite": ["vite@6.4.1", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g=="], - - "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + "vite": ["vite@8.3.4", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", "postcss": "^8.5.29", "rolldown": "~1.2.12", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ=="], } } diff --git a/frontend/package.json b/frontend/package.json index 8793ef05..ab8e600e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,19 +13,19 @@ "dependencies": { "@upsetjs/react": "^1.11.0", "jspdf": "^4.2.1", - "plotly.js-dist-min": "^2.35.2", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "react-router-dom": "^6.28.0", + "plotly.js-dist-min": "^4.1.2", + "react": "^19.3.0", + "react-dom": "^19.3.0", + "react-router-dom": "^7.18.4", "svg2pdf.js": "^2.8.1" }, "devDependencies": { - "@types/plotly.js": "^2.33.4", - "@types/react": "^18.3.18", - "@types/react-dom": "^18.3.5", - "@vitejs/plugin-react": "^4.3.4", - "typescript": "^5.7.2", - "vite": "^6.1.0" + "@types/plotly.js": "^3.0.15", + "@types/react": "^19.3.0", + "@types/react-dom": "^19.3.0", + "@vitejs/plugin-react": "^6.1.2", + "typescript": "^7.0.2", + "vite": "^8.3.4" }, "overrides": { "@types/react": "$@types/react", diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 87457dad..4ea20add 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -439,6 +439,8 @@ export interface DifferentialResult { min_prevalence: number /** nb_glm only. */ offset?: 'tss' | 'rle' + /** nb_glm only: where each taxon's theta comes from. */ + dispersion?: 'ml' | 'glmGamPoi' /** clr_welch only. */ replacement_delta?: number } diff --git a/frontend/src/components/AnalysisChart.tsx b/frontend/src/components/AnalysisChart.tsx index 3e40f157..17e8a437 100644 --- a/frontend/src/components/AnalysisChart.tsx +++ b/frontend/src/components/AnalysisChart.tsx @@ -1,13 +1,14 @@ // © 2026 Joshua Benjamin Jewell. All rights reserved. // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import Plotly from 'plotly.js-dist-min' +import { stripHtmlTags } from '../utils/text' import { AddToReport } from './AddToReport' import { PlotlyChart } from './PlotlyChart' const figureTitle = (fig: unknown): string | null => { const t = (fig as { layout?: { title?: unknown } })?.layout?.title const text = typeof t === 'string' ? t : (t as { text?: string } | undefined)?.text - return text ? text.replace(/<[^>]+>/g, '') : null + return text ? stripHtmlTags(text) : null } // The chart as standalone SVG, at a fixed print width. diff --git a/frontend/src/components/CompositionPanel.tsx b/frontend/src/components/CompositionPanel.tsx index 08863fc3..5e677683 100644 --- a/frontend/src/components/CompositionPanel.tsx +++ b/frontend/src/components/CompositionPanel.tsx @@ -16,6 +16,12 @@ import type { // Fixed colour of the catch-all "Unassigned" bucket, mirroring the backend. const UNASSIGNED_COLOUR = '#95a5a6' +/** + * Browse a run's composition by taxonomy rank or category, with sub-group and + * results-table selectors and controls to save category colours. The analysis + * context's annotation source takes precedence over `source`; absent both, + * VSEARCH is used. A failed table lookup clears the options and shows a toast. + */ export function CompositionPanel({ study, run, group, subgroups, source: configuredSource, }: { diff --git a/frontend/src/components/TaxaCompositionChart.tsx b/frontend/src/components/TaxaCompositionChart.tsx index f14657a3..9601aebd 100644 --- a/frontend/src/components/TaxaCompositionChart.tsx +++ b/frontend/src/components/TaxaCompositionChart.tsx @@ -24,6 +24,13 @@ const LABEL_STYLE: React.CSSProperties = { gap: 4, } +/** + * Select and compute a taxon or category bar chart for one run, or for `runs` + * when that list is non-empty, with optional facets across runs. `table` + * defaults to merged. Providing `subgroup`, including null for all sub-groups, + * controls that selection and hides the internal selector; facets set their + * own scope. Changed inputs clear the figure; chart failures show a toast. + */ export function TaxaCompositionChart({ study, run, diff --git a/frontend/src/figure/charts.ts b/frontend/src/figure/charts.ts index 29845147..6ee30a69 100644 --- a/frontend/src/figure/charts.ts +++ b/frontend/src/figure/charts.ts @@ -2,6 +2,7 @@ // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import { api } from '../api/client' import { extractAlphaPanel } from '../components/alphaMetrics' +import { stripHtmlTags } from '../utils/text' import { textWidth } from './legend' import { PX_PER_PT, tickPt, titlePt, type AlphaMetricKey, type ChartSpec, type FigureDoc, type FigurePane } from './types' @@ -119,7 +120,7 @@ export function paneFigure(raw: unknown, pane: FigurePane, doc: FigureDoc, opts: // A title wider than its pane shrinks to fit, clear of the group letter. const titleRoom = opts.widthPx - 2 * doc.letters.sizePt * PX_PER_PT const titleSize = Math.min(titlePt(doc) * PX_PER_PT, - titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(title.replace(/<[^>]+>/g, ''), titlePt(doc) * PX_PER_PT, family))) + titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(stripHtmlTags(title), titlePt(doc) * PX_PER_PT, family))) for (const k of axisKeys(L, 'x').concat(axisKeys(L, 'y'))) { const ax = { ...(L[k] as Record) } diff --git a/frontend/src/utils/text.test.ts b/frontend/src/utils/text.test.ts new file mode 100644 index 00000000..f5ed71b9 --- /dev/null +++ b/frontend/src/utils/text.test.ts @@ -0,0 +1,24 @@ +// © 2026 Joshua Benjamin Jewell. All rights reserved. +// Licensed under the GNU Affero General Public License version 3 (AGPLv3). +import { describe, expect, test } from 'bun:test' +import { splitLines, stripHtmlTags } from './text' + +describe('stripHtmlTags', () => { + test('removes the markup Plotly allows in a title', () => { + expect(stripHtmlTags('CO2 by run')).toBe('CO2 by run') + }) + + test('a tag hiding inside another does not survive', () => { + expect(stripHtmlTags('ipt>alert(1)')).not.toMatch(/<[^>]+>/) + }) + + test('text without markup, including a bare >, is unchanged', () => { + expect(stripHtmlTags('p > 0.05 for all')).toBe('p > 0.05 for all') + }) +}) + +describe('splitLines', () => { + test('one trimmed value per line, blanks dropped', () => { + expect(splitLines(' a\n\n b \nc')).toEqual(['a', 'b', 'c']) + }) +}) diff --git a/frontend/src/utils/text.ts b/frontend/src/utils/text.ts index dc076655..ffdecc9c 100644 --- a/frontend/src/utils/text.ts +++ b/frontend/src/utils/text.ts @@ -9,3 +9,20 @@ export function splitLines(text: string): string[] { return text.split('\n').map(v => v.trim()).filter(v => v.length > 0) } + +/** + * Strip the pseudo-HTML markup Plotly accepts in titles and axis labels (, + *
, and friends), leaving the bare text for measuring or for a + * filename. A single pass is not enough: replacing "ipt>" once + * removes the inner tag and leaves the halves joined into another, so the + * strip repeats until the text stops changing. + */ +export function stripHtmlTags(text: string): string { + let stripped = text + let previous = '' + do { + previous = stripped + stripped = stripped.replace(/<[^>]+>/g, '') + } while (stripped !== previous) + return stripped +} diff --git a/frontend/src/views/TreeView.tsx b/frontend/src/views/TreeView.tsx index c0778282..8147ac29 100644 --- a/frontend/src/views/TreeView.tsx +++ b/frontend/src/views/TreeView.tsx @@ -183,7 +183,9 @@ function TreeCanvas({ source, tree, src, toast }: { // Autosave the view document. const firstSave = useRef(true) - const saveTimer = useRef() + // React 19's types dropped the zero-argument useRef overload, so the timer + // id starts as an explicit undefined instead. + const saveTimer = useRef(undefined) const saveNow = useCallback(() => { const doc: TreeViewDoc = { version: 1, @@ -228,7 +230,7 @@ function TreeCanvas({ source, tree, src, toast }: { el.addEventListener('wheel', onWheel, { passive: false }) return () => el.removeEventListener('wheel', onWheel) }, []) - const scrollTimer = useRef() + const scrollTimer = useRef(undefined) const onScroll = () => { const el = scrollRef.current if (!el) return diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index e25db006..5aea76c5 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ sourcemap: false, rollupOptions: { output: { - manualChunks: { - plotly: ['plotly.js-dist-min'], - }, + // vite 8 (rolldown) took only the function form; the old object form + // died at "manualChunks is not a function". + manualChunks: (id: string) => (id.includes('plotly.js-dist-min') ? 'plotly' : undefined), }, }, }, diff --git a/install.sh b/install.sh index c5e0a18e..a8e0b718 100755 --- a/install.sh +++ b/install.sh @@ -283,4 +283,4 @@ fi echo "" echo "Running install.jl..." echo "" -julia --project=. install.jl "$@" +julia --project=. scripts/install.jl "$@" diff --git a/launcher.sh b/launcher.sh new file mode 100755 index 00000000..d37ba967 --- /dev/null +++ b/launcher.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# @launcher-deed begin +# ;; SPDX-License-Identifier: MPL-2.0 +# (praxis-deed +# :schema-version "1.0.0" +# :canonical-name "metamanifold-webui-launcher" +# (artefact :type "launcher" :version "0.1.0" +# :generator "hand-adapted from launcher-standard 0.6.0's archetype") +# (app :name "MetaManifold" :display "MetaManifold" +# :url "https://github.com/hyperpolymath/MetaManifold-WebUI" +# :archetype "application") +# (compliance :standard-version "0.6.0" +# :standards ("launcher-standard.adoc")) +# (modes :accepted ("--setup" "--start" "--doctor" "--status" "--agent" "--help" "--version")) +# (platforms :supported ("linux" "macos")) +# (lifecycle-phases :covered ("provision" "diagnose" "run") +# :deferred ("heal"))) +# @launcher-deed end +# +# The MetaManifold launcher, in the launcher-standard's mode vocabulary but +# self-contained: this is an application repository, not an RSR-minted one, +# so the modes delegate directly to the repository's own entry points +# (install.sh, start.sh) rather than through provision-set machinery. +# +# The one thing that is this launcher's own is --agent: the git-signing +# agent's persistence is the operator's choice, and every mode of it is +# reversible. The signing key is ~/.ssh/git-signing2; the socket is +# /tmp/git-signing-agent.sock. +# +# --agent systemd a systemd user service owns the agent, and the shell +# profile exports the socket (a service cannot export +# into login shells). Survives reboots; one instance. +# --agent profile the shell profile starts the agent if the socket is +# dead and exports it. No systemd; works over plain ssh. +# --agent both the service plus the spawn-if-dead fallback line. +# --agent none removes whatever of the above was installed. +# --agent status reports socket, key, and which mechanisms are present. +# +# Each mechanism is written between marked blocks in the profile and a unit +# file, idempotently, with a dated backup of anything edited. + +set -uo pipefail + +REPO_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +SOCKET="/tmp/git-signing-agent.sock" +KEY="${HOME}/.ssh/git-signing2" +UNIT_NAME="git-signing-agent" +UNIT_FILE="${HOME}/.config/systemd/user/${UNIT_NAME}.service" +PROFILE="${HOME}/.profile" +BLOCK_BEGIN="# >>> metamanifold git-signing agent >>>" +BLOCK_END="# <<< metamanifold git-signing agent <<<" + +# Print all arguments as one launcher-prefixed message to standard output. +say() { printf 'launcher: %s\n' "$*"; } + +# Print all arguments as one launcher-prefixed message to standard error. +warn() { printf 'launcher: %s\n' "$*" >&2; } + +# Report all arguments through warn and terminate the launcher with status 1. +die() { warn "$*"; exit 1; } + +# Return 0 only when SOCKET is a Unix socket and ssh-add can list identities. +# An agent with no loaded identities also returns non-zero. +agent_alive() { + [ -S "$SOCKET" ] && SSH_AUTH_SOCK="$SOCKET" ssh-add -l >/dev/null 2>&1 +} + +# Remove marked agent blocks from PROFILE, following symlinks and preserving +# file permissions. A missing profile succeeds. Return 1 for nested or +# unterminated blocks, or file-operation failures; no backup is made here. +remove_profile_block() { + [ -f "$PROFILE" ] || return 0 + local target tmp mode + target="$(readlink -f "$PROFILE")" || return 1 + tmp="$(mktemp "$(dirname "$target")/.profile.tmp.XXXXXX")" || return 1 + if ! awk -v b="$BLOCK_BEGIN" -v e="$BLOCK_END" ' + $0 == b { + if (skip) exit 1 + skip = 1 + next + } + $0 == e { + if (skip) { skip = 0; next } + print + next + } + !skip { print } + END { if (skip) exit 1 } + ' "$target" > "$tmp"; then + rm -f "$tmp" + return 1 + fi + mode="$(stat -c '%a' "$target" 2>/dev/null || stat -f '%Lp' "$target")" || { + rm -f "$tmp" + return 1 + } + chmod "$mode" "$tmp" || { + rm -f "$tmp" + return 1 + } + mv "$tmp" "$target" || { + rm -f "$tmp" + return 1 + } +} + +# Back up PROFILE and replace its marked agent block, creating the file if absent. +# "export" sets the socket for future shells; "spawn" also tries to start an +# agent when ssh-add cannot list identities. Neither mode loads the signing key. +# Exit 1 if profile creation, backup or block removal fails; append failures +# return a non-zero status and do not restore the backup. +write_profile_block() { # $1 = "export" | "spawn" + [ -f "$PROFILE" ] || touch "$PROFILE" || die "could not create $PROFILE" + local backup + backup="$(mktemp "$PROFILE.launcher-backup.$(date +%Y%m%d%H%M%S).XXXXXX")" || + die "could not create backup of $PROFILE" + cp -p "$PROFILE" "$backup" || { + rm -f "$backup" + die "backup of $PROFILE failed" + } + remove_profile_block || die "could not update $PROFILE" + { + printf '%s\n' "$BLOCK_BEGIN" + if [ "$1" = "spawn" ]; then + printf 'if [ ! -S "%s" ] || ! SSH_AUTH_SOCK="%s" ssh-add -l >/dev/null 2>&1; then\n' "$SOCKET" "$SOCKET" + printf ' ssh-agent -a "%s" >/dev/null 2>&1\n' "$SOCKET" + printf 'fi\n' + fi + printf 'export SSH_AUTH_SOCK="%s"\n' "$SOCKET" + printf '%s\n' "$BLOCK_END" + } >> "$PROFILE" +} + +# Create or overwrite UNIT_FILE for an ssh-agent on SOCKET, without a backup. +# Return the write status; this does not enable or start the service. +write_unit() { + mkdir -p "$(dirname "$UNIT_FILE")" + cat > "$UNIT_FILE" </dev/null || die "systemctl not found; use --agent profile" + write_unit + systemctl --user daemon-reload + systemctl --user enable --now "$UNIT_NAME" || die "could not enable ${UNIT_NAME} (try: loginctl enable-linger)" + write_profile_block export + say "systemd user service enabled; profile exports SSH_AUTH_SOCK=$SOCKET" + say "note: over plain ssh, or without a user session, also run: loginctl enable-linger \$USER" + ;; + profile) + write_profile_block spawn + say "profile now starts the agent if the socket is dead and exports SSH_AUTH_SOCK=$SOCKET" + ;; + both) + command -v systemctl >/dev/null || die "systemctl not found; use --agent profile" + write_unit + systemctl --user daemon-reload + systemctl --user enable --now "$UNIT_NAME" || die "could not enable ${UNIT_NAME}" + write_profile_block spawn + say "systemd service plus spawn-if-dead profile fallback installed" + ;; + none) + if command -v systemctl >/dev/null 2>&1; then + systemctl --user disable --now "$UNIT_NAME" >/dev/null 2>&1 || + warn "could not disable $UNIT_NAME" + else + warn "systemctl not found; user unit was not disabled" + fi + rm -f "$UNIT_FILE" + if command -v systemctl >/dev/null 2>&1; then + systemctl --user daemon-reload || + warn "could not reload the user manager" + fi + if [ -f "$PROFILE" ]; then + backup="$(mktemp "$PROFILE.launcher-backup.$(date +%Y%m%d%H%M%S).XXXXXX")" || { + warn "could not create backup of $PROFILE; profile block was not removed" + return 1 + } + if ! cp -p "$PROFILE" "$backup"; then + rm -f "$backup" + warn "could not back up $PROFILE; profile block was not removed" + return 1 + fi + fi + if ! remove_profile_block; then + warn "could not remove the profile block" + return 1 + fi + say "removed unit and profile block; any agent already running is left alone" + ;; + status) + agent_alive && say "socket alive: $SOCKET" || say "socket dead or absent: $SOCKET" + [ -f "$KEY" ] && say "key present: $KEY" || say "key MISSING: $KEY" + [ -f "$UNIT_FILE" ] && say "systemd unit: installed" || say "systemd unit: not installed" + grep -q "$BLOCK_BEGIN" "$PROFILE" 2>/dev/null && say "profile block: present" || say "profile block: absent" + ;; + *) + die "usage: launcher.sh --agent systemd|profile|both|none|status" + ;; + esac +} + +# Report toolchain and signing readiness, then exit 1 if julia, bun, R or git +# is missing from PATH, or agent_alive fails; otherwise exit 0. Missing tool +# configuration or a signing-key check failure only produces a warning. +doctor() { + fail=0 + for tool in julia bun R git; do + if command -v "$tool" >/dev/null; then + say "ok: $tool on PATH" + else + warn "MISSING: $tool not on PATH" + fail=1 + fi + done + [ -f "$REPO_DIR/config/tools.yml" ] && say "ok: config/tools.yml exists" \ + || warn "config/tools.yml missing (run: ./launcher.sh --setup)" + if agent_alive; then + say "ok: signing agent alive at $SOCKET" + SSH_AUTH_SOCK="$SOCKET" ssh-add -l 2>/dev/null | grep -q "$(ssh-keygen -lf "${KEY}.pub" 2>/dev/null | awk '{print $2}')" \ + && say "ok: signing key loaded" || warn "signing key not loaded (run: ssh-add $KEY)" + else + warn "signing agent not reachable at $SOCKET (see: ./launcher.sh --agent status)" + fail=1 + fi + exit "$fail" +} + +# Print the launcher header and available modes to standard output. +help() { + sed -n '2,40p' "${BASH_SOURCE[0]}" | grep '^#' | sed 's/^# \{0,1\}//' + cat <<'MODES' +Modes: + --setup install everything (delegates to install.sh; args pass through) + --start serve the web UI (delegates to start.sh) + --doctor check the toolchain and the signing agent + --status whether the server looks reachable on its port + --agent MODE systemd | profile | both | none | status (the signing agent) + --help this text + --version the launcher's version +MODES +} + +# Print the launcher and launcher-standard versions to standard output. +version() { echo "metamanifold-webui-launcher 0.1.0 (launcher-standard 0.6.0, hand-adapted)"; } + +# Dispatch the first argument as a launcher mode. Setup replaces this process +# with install.sh and forwards remaining arguments; start executes start.sh. +# Status checks the configured local port (default 8080) with a two-second +# timeout; HTTP error responses count as reachable and connection failure does +# not set a failure exit status. Missing or unknown modes print help and exit 1. +main() { + case "${1:-}" in + --setup) shift; exec bash "$REPO_DIR/install.sh" "$@" ;; + --start) exec bash "$REPO_DIR/start.sh" ;; + --doctor) doctor ;; + --status) + status_url="http://127.0.0.1:${JULIA_METAMANIFOLD_PORT:-8080}/" + if curl -s -o /dev/null --max-time 2 "$status_url"; then + say "something is answering on $status_url" + else + say "nothing answering on $status_url (start with: ./launcher.sh --start)" + fi + ;; + --agent) shift; agent_mode "${1:-}" ;; + --help|-h) help ;; + --version) version ;; + *) help; exit 1 ;; + esac +} + +main "$@" diff --git a/proofs/agda/MetaManifold/All.agda b/proofs/agda/MetaManifold/All.agda index 7cedd39a..0770f154 100644 --- a/proofs/agda/MetaManifold/All.agda +++ b/proofs/agda/MetaManifold/All.agda @@ -12,3 +12,5 @@ module MetaManifold.All where import MetaManifold.Prelude import MetaManifold.BenjaminiHochberg import MetaManifold.Scaling +import MetaManifold.Permanova +import MetaManifold.Significance diff --git a/proofs/agda/MetaManifold/Permanova.agda b/proofs/agda/MetaManifold/Permanova.agda new file mode 100644 index 00000000..fbf0220a --- /dev/null +++ b/proofs/agda/MetaManifold/Permanova.agda @@ -0,0 +1,473 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- PERMANOVA's permutation bookkeeping: the two things `run_permanova` in +-- `src/analysis/analysis.jl` decides for itself rather than taking from vegan +-- (branch fix/permanova-terms, PR #35), proved. +-- +-- 1. `min_p_value = 1 / (n_perm + 1)`. vegan's permutation p-value is +-- (k + 1)/(B + 1), where B is the number of permutations performed and k +-- how many of them gave a statistic at least the observed one (0 ≤ k ≤ B). +-- The theorems say that 1/(B + 1) is the smallest p-value B permutations +-- can produce, that it is attained, and that no threshold below it can be +-- reached. +-- 2. Which terms have their p-value withheld. With permutations restricted +-- to blocks, Julia withholds the p-value of a term when +-- `_varies_within_any_block(values, blocks)` is false. The theorems say +-- that this happens exactly when every within-block permutation leaves the +-- term's labels where they were, so the permutations never exchange them. +-- They also say that when it is true, some within-block permutation does +-- exchange them. So no testable term is withheld, and no untestable term +-- is reported. +-- +-- What is NOT modelled, and why: +-- +-- * The pseudo-F statistic and vegan's distribution of it. The p-value +-- theorems take k as given; they are about the arithmetic of (k + 1)/(B + 1), +-- not about which permutations exceed the observed F. +-- * How many within-block permutations exist. B is whatever `permute` +-- reports (`attr(perm_res, "control")$nperm`). Six blocks of two samples +-- admit 2⁶ = 64 arrangements, the identity and 63 others, which is the 63 in +-- test_permanova.jl. That count is the product of the blocks' factorials; +-- it is not proved here. +-- * Unblocked runs. Julia only asks `_varies_within_any_block` when some +-- block label repeats (`blocked`). Without repeated blocks the +-- permutations are unrestricted and nothing is withheld; that branch is +-- Julia's `if`, not a theorem. +-- * Julia's `isequal` on NaN or `missing`. Equality of labels and of blocks +-- is an arbitrary decidable equality here. +-- * Floating point. `1 / (n_perm + 1)` is exact here and a Float64 in Julia. +-- +-- Theorems (names as in proofs/agda/README.md): +-- +-- minP-attained (0 + 1)/(B + 1) is 1/(B + 1) +-- p-floor every permutation p-value is at least 1/(B + 1) +-- p-at-most-one ... and at most 1 when k ≤ B +-- below-floor-unreachable a threshold below 1/(B + 1) is below every p-value +-- minP-anti more permutations never raise the floor +-- minP-63 with 63 permutations the floor is 1/64 +-- no-p-at-1%-with-63 with 63 permutations no p-value is at or below 0.01 +-- never-exchanged a block-constant term is fixed by every +-- within-block permutation +-- exchanged a term with two labels in one block is moved by a +-- within-block permutation (a transposition) +-- untestable⇔fixed block-constant ⇔ fixed by every within-block +-- permutation +-- varies-false⇔Constant the model of `_varies_within_any_block` returns +-- false exactly on block-constant input +-- withheld⇔never-exchanged ... so its false answer, which withholds the +-- p-value, holds exactly when no within-block +-- permutation exchanges the term's labels +-- tested⇔exchanged ... and its true answer, which reports the p-value, +-- holds exactly when some within-block permutation +-- does +-- constant-in-blocks, varies-in-a-block +-- the two direct cases in test_permanova.jl +-- fixture-group-withheld, fixture-run-tested +-- the twelve-sample fixture in test_permanova.jl: group +-- is never exchanged, run is + +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.Permanova where + +open import Data.Bool.Base using (Bool; true; false) +open import Data.Empty using (⊥-elim) +open import Data.Fin.Base using (Fin) +open import Data.Fin.Properties using (_≟_) +open import Data.Fin.Permutation using (Permutation′; _⟨$⟩ʳ_; transpose) +open import Data.Integer.Base as ℤ using (+_; +[1+_]) +import Data.Integer.Properties as ℤₚ +open import Data.List.Base using (List; []; _∷_; _++_; [_]; length; lookup) +open import Data.List.Membership.Propositional using (_∈_) +open import Data.List.Membership.Propositional.Properties + using (∈-lookup; ∈-++⁺ˡ; ∈-++⁺ʳ; ∈-++⁻) +open import Data.List.Properties using (++-assoc; ++-identityʳ) +open import Data.List.Relation.Unary.Any using (here; index) +open import Data.List.Relation.Unary.Any.Properties using (lookup-index) +open import Data.Maybe.Base using (Maybe; just; nothing) +open import Data.Nat.Base as ℕ using (ℕ; suc; z≤n; s≤s) +import Data.Nat.Properties as ℕₚ +open import Data.Product.Base using (Σ; ∃₂; _×_; _,_; proj₁; proj₂) +open import Data.Rational.Base as ℚ using (ℚ; _/_; 1ℚ; _≤_; _<_) +import Data.Rational.Properties as ℚₚ +open import Data.Rational.Unnormalised.Base using (mkℚᵘ; *≤*) +open import Data.Sum.Base using (inj₁; inj₂) +open import Function.Bundles using (_⇔_; mk⇔; Equivalence) +open import Relation.Binary.Definitions using (DecidableEquality) +open import Relation.Binary.PropositionalEquality + using (_≡_; _≢_; refl; sym; trans; cong; subst) +open import Relation.Nullary using (¬_; yes; no) +open import Relation.Nullary.Decidable using (toWitness) +open import Relation.Nullary.Negation using (contradiction) + +open import MetaManifold.Prelude using (numerator-monotone-≤; fromℚᵘ-mono-≤) + +------------------------------------------------------------------------ +-- 1. The smallest p-value B permutations can produce + +-- vegan's permutation p-value after B permutations, k of which gave a +-- statistic at least the observed one. +permP : ℕ → ℕ → ℚ +permP B k = + suc k / suc B + +-- Julia's `min_p_value = 1 / (Int(n_perm) + 1)`. +minP : ℕ → ℚ +minP B = + 1 / suc B + +-- The floor is a p-value: the one reported when no permutation reaches the +-- observed statistic. +minP-attained : ∀ B → permP B 0 ≡ minP B +minP-attained B = refl + +-- No permutation p-value is below the floor. +p-floor : ∀ B k → minP B ≤ permP B k +p-floor B k = fromℚᵘ-mono-≤ {mkℚᵘ (+ 1) B} {mkℚᵘ (+ suc k) B} + (numerator-monotone-≤ (+ 1) (+ suc k) B (ℤ.+≤+ (s≤s z≤n))) + +-- With k ≤ B, none is above 1. +p-at-most-one : ∀ B k → k ℕ.≤ B → permP B k ≤ 1ℚ +p-at-most-one B k k≤B = fromℚᵘ-mono-≤ {mkℚᵘ (+ suc k) B} {mkℚᵘ (+ 1) 0} + (*≤* (ℤₚ.≤-trans (ℤₚ.≤-reflexive (ℤₚ.*-identityʳ (+ suc k))) + (ℤₚ.≤-trans (ℤ.+≤+ (s≤s k≤B)) + (ℤₚ.≤-reflexive (sym (ℤₚ.*-identityˡ +[1+ B ])))))) + +-- A significance threshold below the floor is below every p-value, so no term +-- can be significant at it, whatever the data. +below-floor-unreachable : ∀ B k {α} → α < minP B → α < permP B k +below-floor-unreachable B k α +-- +-- The significance reporting discipline: what a test that did not run may be +-- read as, proved. `run_alpha_significance` in `src/analysis/analysis.jl` +-- returns an `AlphaSignificance` whose status distinguishes a computed +-- omnibus result from an unavailable or failed test (`:r_unavailable`, +-- `:no_paired_samples`, `:test_failed`), and whose docstring carries the +-- rule: "False means no test was performed, so neither `omnibus` nor +-- `pairwise` may be presented as a finding." +-- +-- The theorems here apply the Observational Null Disclosure discipline +-- (hyperpolymath/absolute-zero, the OND pillar) to that rule: +-- +-- 1. `publish` reads the omnibus p-value only from a result whose test +-- ran, and requires evidence that it ran. On a result that did not run, +-- the evidence would be an inhabitant of `false ≡ true`, which does not +-- exist: the branch cannot produce a number, refused by the typechecker +-- -- not asserted by a test. +-- 2. `pairwise-published` is the same for the pairwise table. +-- 3. `reason` is the dual: the reader-facing explanation can be read only +-- from a result that did not run. +-- 4. `not-computed⇒unrepresentable`: a result whose status is not +-- `computed` carries no numbers by construction -- there is no field to +-- read, so an unrun test can never be published as a null result. +-- +-- Mapped to the Julia (test/unit/test_r_runtime.jl): "a test that did not +-- run is not a null result", "a chart states that pairwise tests were not +-- run", "the caption distinguishes the reasons a test did not run", and "an +-- omnibus statistic R cannot compute is a failed test". +-- +-- What is modelled, and what is not: the statuses are Julia's; the omnibus +-- p-value is an exact rational (the Julia holds a Float64); the pairwise +-- table is modelled as the list of its p-values, not the DataFrame; the +-- reason strings are opaque. The statistics themselves are NOT modelled -- +-- this module is about what may be read from a result, not how the result +-- was computed. + +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.Significance where + +open import Data.Bool using (Bool; true; false) +open import Data.List using (List; map) +open import Data.Rational using (ℚ) +open import Data.String using (String) +open import Relation.Nullary using (¬_) +open import Relation.Binary.PropositionalEquality using (_≡_; refl; cong) + +-- Julia's statuses: :computed, :r_unavailable, :no_paired_samples, :test_failed. +data Status : Set where + computed r-unavailable no-paired-samples test-failed : Status + +-- The discipline: the omnibus p-value and the pairwise p-values exist only +-- in the constructor that claims the test ran. A result that reveals nothing +-- names its status, carries a proof that the status is not `computed`, and a +-- reader-facing reason -- and no numbers, because there are none to read. +data AlphaSignificance : Set where + computed-result : (omnibus : ℚ) (pairwise : List ℚ) → AlphaSignificance + reveals-nothing : (status : Status) → ¬ (status ≡ computed) → String → AlphaSignificance + +-- Julia's was_computed, mirrored. +was-computed : AlphaSignificance → Bool +was-computed (computed-result _ _) = true +was-computed (reveals-nothing _ _ _) = false + +-- 1. Publishing the omnibus p-value requires evidence the test ran. On the +-- reveals-nothing branch the evidence is an inhabitant of `false ≡ true`, +-- which does not exist, so the branch cannot produce a number at all. +publish : (r : AlphaSignificance) → was-computed r ≡ true → ℚ +publish (computed-result omnibus _) refl = omnibus +publish (reveals-nothing _ _ _) () + +-- 2. The pairwise table, same discipline. +pairwise-published : (r : AlphaSignificance) → was-computed r ≡ true → List ℚ +pairwise-published (computed-result _ pairwise) refl = pairwise +pairwise-published (reveals-nothing _ _ _) () + +-- 3. The dual: the reader-facing reason can be read only from a result that +-- did not run. A computed result has no reason to show, and the evidence +-- it would need (`true ≡ false`) does not exist. +reason : (r : AlphaSignificance) → was-computed r ≡ false → String +reason (computed-result _ _) () +reason (reveals-nothing _ _ why) refl = why + +-- 4. The status of a result that did not run is never `computed`, by +-- construction: the proof obligation lives in the constructor, so a +-- `reveals-nothing` value cannot even be built with the `computed` +-- status. +unrun-status-not-computed : + (s : Status) (h : ¬ (s ≡ computed)) (why : String) → + let r = reveals-nothing s h why in was-computed r ≡ false +unrun-status-not-computed s h why = refl + +-- The status of a computed result is `computed`, and its evidence is the +-- constructor's own shape -- so the two constructors never meet: a result +-- is either readable as numbers or readable as a reason, never both. +computed-status : (o : ℚ) (p : List ℚ) → + was-computed (computed-result o p) ≡ true +computed-status o p = refl + +-- Map every pairwise p-value through a function without changing the +-- publish discipline: reading remains possible only on the computed branch. +map-pairwise : (f : ℚ → ℚ) (r : AlphaSignificance) → was-computed r ≡ true → List ℚ +map-pairwise f (computed-result _ pairwise) refl = map f pairwise +map-pairwise f (reveals-nothing _ _ _) () diff --git a/proofs/agda/README.md b/proofs/agda/README.md index 304738cc..2590a5ad 100644 --- a/proofs/agda/README.md +++ b/proofs/agda/README.md @@ -3,7 +3,7 @@ SPDX-License-Identifier: AGPL-3.0-only SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --> -# Machine-checked proofs: Benjamini–Hochberg and size-factor scaling +# Machine-checked proofs: Benjamini–Hochberg, size-factor scaling and PERMANOVA permutations `proofs/agda/` holds Agda proofs of the exact-arithmetic properties of three functions in `src/analysis/differential.jl`: `bh_adjust`, `tss_factors` and @@ -12,6 +12,12 @@ differential-abundance change (branch `feat/differential-abundance-nb`, commits `6c3e8b2` and `23dc89c`). These proofs depend on that change being merged first. +`MetaManifold/Permanova.agda` covers the two things `run_permanova` in +`src/analysis/analysis.jl` decides for itself rather than taking from vegan: +the reported `min_p_value`, and which terms have their p-value withheld +(`_varies_within_any_block`). Both arrive with the PERMANOVA change (branch +`fix/permanova-terms`, PR #35), which these proofs depend on. + Every module is checked with `--safe --without-K`. There are no postulates, no holes and no termination or positivity pragmas. `proofs/tests/axiom-audit.sh` enforces this. It also fails if any module is @@ -62,6 +68,18 @@ vector. - **Input validation** (finite and in `[0, 1]`) is the hypothesis `All Valid ps`. The Julia `ArgumentError` paths are therefore assumptions of the theorems, not theorems. +- **`variesWithinAnyBlock`** follows the shape of + `_varies_within_any_block`. A list of `(value, block)` pairs stands in for + `zip(values, blocks)` and an association list for the `first_seen` + dictionary. As with `get!`, a block's value is stored only the first time + the block is seen, and the loop returns `true` at the first sample whose + value differs from its block's stored one. Equality of values and of blocks + is any decidable equality, standing in for `isequal`. +- **Within-block permutations** are permutations of the samples + (`Permutation′ n` from the standard library) that keep every sample in its + block. A term's labels are a function from samples to values. The + p-value is vegan's `(k + 1)/(B + 1)` for `B` permutations, `k` of which + reached the observed statistic. ## Theorem map @@ -90,6 +108,43 @@ proportion theorems are the relative-abundance reading of TSS, proved because the requirement names them. They do not describe a value the Julia code returns. +### PERMANOVA (`Permanova`) + +| Agda theorem | Plain statement | Julia | Test (`test/unit/test_permanova.jl`) | +|---|---|---|---| +| `minP-attained`, `p-floor` | `1/(B + 1)` is the smallest p-value `B` permutations can give, and it is given when no permutation reaches the observed statistic. | `min_p_value = 1 / (Int(n_perm) + 1)` | lines 73, 84 | +| `p-at-most-one` | With `k ≤ B`, no p-value is above 1. | (vegan) | – | +| `below-floor-unreachable` | A threshold below `1/(B + 1)` is below every p-value, so no term can meet it. | `min_p_value` | – | +| `minP-anti` | More permutations never raise the floor. | – | – | +| `minP-63`, `no-p-at-1%-with-63` | With 63 permutations the floor is `1/64`, and no p-value is at or below 0.01. | – | lines 83–84 | +| `never-exchanged` | A term that is constant within every block is left in place by every within-block permutation, so its labels are never exchanged. | the reason for withholding | – | +| `exchanged` | A term with two values in one block is moved by a within-block permutation: the swap of those two samples. | – | – | +| `untestable⇔fixed` | Constant within every block ⇔ fixed by every within-block permutation. | – | – | +| `varies-false⇔Constant` | The model of `_varies_within_any_block` returns `false` exactly when no block holds two different values. | `_varies_within_any_block` | lines 47–51 | +| `withheld⇔never-exchanged` | So `untestable` (the function returned `false`) holds exactly when no within-block permutation exchanges the term's labels. | `untestable`, `withheld` | lines 92–93 | +| `tested⇔exchanged` | And a p-value is reported (the function returned `true`) exactly when some within-block permutation does. No testable term is withheld. | `untestable`, `withheld` | line 98 | +| `constant-in-blocks`, `varies-in-a-block` | The two direct cases in the test, by evaluation. | `_varies_within_any_block` | lines 49–50 | +| `fixture-group-withheld`, `fixture-run-tested` | On the test's twelve samples (six individuals, two runs each, three individuals per group), `group` is never exchanged and `run` is. | `run_permanova` | lines 83–98 | + +### Significance reporting (`Significance`) + +The Observational Null Disclosure discipline (hyperpolymath/absolute-zero, OND +pillar) applied to `run_alpha_significance`'s reporting rule: a test that did +not run reveals nothing, and must never be published as a null result. The +numbers exist only in the constructor whose status is `computed`; every other +status carries a proof it is not `computed` and a reader-facing reason. + +| Agda theorem | Plain statement | Julia | Test (`test/unit/test_r_runtime.jl`) | +|---|---|---|---| +| `publish` | The omnibus p-value is readable only from a result whose test ran, with evidence it ran; the unrun branch cannot produce a number (the typechecker refuses it). | `was_computed` guards | lines 100–124 | +| `pairwise-published` | The pairwise table, same discipline. | `was_computed` guards | lines 125–152 | +| `reason` | The dual: the reader-facing explanation is readable only from a result that did not run. | the `reason` field | lines 153–175 | +| `unrun-status-not-computed`, `computed-status` | A result is readable as numbers or as a reason, never both: the two constructors cannot meet. | the `AlphaSignificance` struct | lines 176+ | + +What is modelled: the four statuses exactly; the omnibus p-value as an exact +rational; the pairwise table as its list of p-values. What is not: the +statistics themselves, and the DataFrame shape. + ## Corrections to the requirement wording - **R-BH-2a is true for the procedure.** An earlier archived note recorded @@ -118,6 +173,16 @@ returns. proved. - **Floating point.** Every statement is exact. `Float64` rounding, and a refusal caused by underflow to `0.0`, are outside the model. +- **PERMANOVA: the pseudo-F statistic, and how many within-block + permutations exist.** The p-value theorems take `B` and `k` as given. `B` + is what `permute` reports, and `k` depends on the F distribution, neither of + which is modelled. Six blocks of two samples admit `2⁶ = 64` + arrangements, the identity and 63 others, which is the 63 in the test. + That count (the product of the blocks' factorials) is not proved. +- **PERMANOVA: the unblocked case.** Julia asks `_varies_within_any_block` + only when some block label repeats (`blocked`). Otherwise the permutations + are unrestricted and nothing is withheld; that is Julia's `if`, not a + theorem. - **Equivalence of the Agda and Julia code** is argued, not formalised. The model mirrors the loop, and the two differences above are each discharged by a proved lemma plus a short argument. @@ -138,3 +203,17 @@ returns. A concrete counterexample is `p = [0.01, 0.011]`, `n = 2`. Without the minimum the scaled values are `0.02` and `0.011`, which are not monotone. + +3. **The PERMANOVA proofs refuse a wrong model.** Each of these, planted in + a scratch copy of `Permanova.agda`, made Agda reject it: + - `variesWithinAnyBlock` never stores a block's first value; + - it returns `true` on a match instead of a mismatch; + - it never returns `true`; + - the p-value drops the `+ 1` from `k + 1`; + - the floor is `1/(B + 2)`; + - any permutation counts as within-block; + - the fixture's `group` differs inside one individual (then + `fixture-group-withheld` fails); + - the fixture's `run` is constant inside every individual (then + `fixture-run-tested` fails); + - the 1% claim is made at 2% (`1/50 > 1/64`). diff --git a/renv.lock b/renv.lock index dfac546e..a487543c 100644 --- a/renv.lock +++ b/renv.lock @@ -1668,65 +1668,6 @@ "NeedsCompilation": "yes", "Repository": "CRAN" }, - "dplyr": { - "Package": "dplyr", - "Version": "1.2.1", - "Source": "Repository", - "Type": "Package", - "Title": "A Grammar of Data Manipulation", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Romain\", \"François\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Lionel\", \"Henry\", role = \"aut\"), person(\"Kirill\", \"Müller\", role = \"aut\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4777-038X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A fast, consistent tool for working with data frame like objects, both in memory and out of memory.", - "License": "MIT + file LICENSE", - "URL": "https://dplyr.tidyverse.org, https://github.com/tidyverse/dplyr", - "BugReports": "https://github.com/tidyverse/dplyr/issues", - "Depends": [ - "R (>= 4.1.0)" - ], - "Imports": [ - "cli (>= 3.6.2)", - "generics", - "glue (>= 1.3.2)", - "lifecycle (>= 1.0.5)", - "magrittr (>= 1.5)", - "methods", - "pillar (>= 1.9.0)", - "R6", - "rlang (>= 1.1.7)", - "tibble (>= 3.2.0)", - "tidyselect (>= 1.2.0)", - "utils", - "vctrs (>= 0.7.1)" - ], - "Suggests": [ - "broom", - "covr", - "DBI", - "dbplyr (>= 2.2.1)", - "ggplot2", - "knitr", - "Lahman", - "lobstr", - "nycflights13", - "purrr", - "rmarkdown", - "RSQLite", - "stringi (>= 1.7.6)", - "testthat (>= 3.1.5)", - "tidyr (>= 1.3.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/build/compilation-database": "true", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre] (ORCID: ), Romain François [aut] (ORCID: ), Lionel Henry [aut], Kirill Müller [aut] (ORCID: ), Davis Vaughan [aut] (ORCID: ), Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, "farver": { "Package": "farver", "Version": "2.1.2", diff --git a/scripts/check-occupancy-grades.jl b/scripts/check-occupancy-grades.jl new file mode 100755 index 00000000..ce160d09 --- /dev/null +++ b/scripts/check-occupancy-grades.jl @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Gate for the graded reading of the recorded run (fork issue #105): rebuild each +# order's timeline from its sample window (docs/occupancy/schedules.yml), grade it +# with the module the probe itself uses (src/core/memory_probe.jl — included +# rather than the whole package, so this needs OrderedCollections and YAML only), +# and compare the algebra's predicted peak with the measured one. +# +# The recorded run is fixed evidence, so the comparison is an equality: this is +# the reading docs/occupancy-2026-10.adoc publishes, and a change that moves it +# must be a deliberate one. Two further claims hold of every order and are +# checked: the prediction is never above the cost bound (occupancy≤cost) and the +# slack between them is never negative (what the cuts reclaim). +# +# Run from the repository root: julia --project=. scripts/check-occupancy-grades.jl +include(joinpath(@__DIR__, "..", "src", "core", "memory_probe.jl")) +using .MemoryProbe +using YAML + +const ROOT = normpath(joinpath(@__DIR__, "..")) +const RESULTS = joinpath(ROOT, "docs", "occupancy", "results-2026-10-14.yml") +const SCHEDULES = joinpath(ROOT, "docs", "occupancy", "schedules.yml") + +orders = YAML.load_file(RESULTS)["orders"] +schedules = YAML.load_file(SCHEDULES) + +failures = 0 +function fail(message) + global failures += 1 + println("FAIL: ", message) +end + +println(rpad("order", 16), rpad("predicted", 11), rpad("measured", 11), + rpad("cost", 8), "reclaimed") +# An order's timeline, from the peaks the probe recorded and the window each task +# ran in. Nothing else is taken from the schedule: the grades are the +# measurement's. +function timeline(rec, windows) + tasks = Any[Dict{String,Any}("label" => t["label"], + "peak_rss_kib" => t["peak_rss_kib"], + "first_sample" => windows[i][1], + "last_sample" => windows[i][2]) + for (i, t) in enumerate(rec["tasks"])] + Dict{String,Any}("child_peak_rss_kib" => rec["child_peak_rss_kib"], "tasks" => tasks) +end + +for name in sort(collect(keys(orders))) + rec = orders[name] + occ = timeline(rec, schedules[name]) + predicted = MemoryProbe.predicted_peak_rss_kib(occ) + cost = MemoryProbe.cost_bound_rss_kib(occ) + reclaimed = MemoryProbe.reclaimed_kib(occ) + measured = rec["child_peak_rss_kib"] + println(rpad(name, 16), rpad(string(predicted), 11), rpad(string(measured), 11), + rpad(string(cost), 8), string(reclaimed)) + + predicted == measured || + fail("$name: predicted $predicted KiB != measured $measured KiB") + predicted <= cost || + fail("$name: predicted $predicted KiB above the cost bound $cost KiB (occupancy≤cost)") + reclaimed >= 0 || + fail("$name: reclaimed $reclaimed KiB is negative") + d = MemoryProbe.disagreement(occ) + d === nothing || fail("$name: $d") +end + +if failures > 0 + println("check-occupancy-grades: $failures failure(s)") + exit(1) +end +println("check-occupancy-grades: OK — the prediction reproduces the recorded measurement") diff --git a/scripts/check-occupancy.sh b/scripts/check-occupancy.sh new file mode 100755 index 00000000..c43f28e4 --- /dev/null +++ b/scripts/check-occupancy.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Gate for the occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): +# 1. each Session IR text checks, and occupancy-types' stepper counts its bytes; +# 2. TypedPeak.agda and GradedStages.agda type-check against the pinned +# siblings, and each module in docs/occupancy/reject is refused at its own +# declaration; +# 3. the counted bytes equal the typed peak, order by order. +# Needs julia and agda (2.6.4.x) on PATH. Exit 0 only if all three hold. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +export LC_ALL="${LC_ALL:-C.UTF-8}" + +bash scripts/fetch-siblings.sh +deps=_build/dependencies +sir="$deps/occupancy-types/src/julia" +agda --version | head -1 +flags=(--safe --without-K --no-libraries --ignore-interfaces --double-check + -i docs/occupancy -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") +logs=_build/check-logs +mkdir -p "$logs" +failures=0 + +# GitHub renders an `::error` line as a check annotation, and annotations -- unlike +# job logs -- can be read back through the API. A failure that only happens in CI +# must be readable from a machine that cannot download the log, so every failure +# below says what it was and repeats the prover's own last lines: % escapes to +# %25 and newlines to %0A, as the workflow-command syntax requires. +annotate() { # title file log + [ -n "${GITHUB_ACTIONS:-}" ] || return 0 + printf '::error file=%s,title=%s::%s\n' "$2" "$1" \ + "$(awk 'NR > 1 { printf "%%0A" } { gsub(/%/, "%25"); printf "%s", $0 }' "$3" | + tail -c 6000)" +} + +# One Agda module that must type-check, with its own log. +check_agda() { # file label + if ! agda "${flags[@]}" "$1" >"$logs/$2.log" 2>&1; then + cat "$logs/$2.log" + annotate "Agda refused $2" "$1" "$logs/$2.log" + echo "ERROR: $1 does not type-check" >&2 + exit 1 + fi + echo "PASS: $2" +} + +# Order -> Session IR file and TypedPeak lemma. +orders=(asv-then-otu otu-then-asv concurrent) + +echo "== Session IR (occupancy-types v0 stepper)" +declare -A counted +for o in "${orders[@]}"; do + file="docs/occupancy/vsearch-$o.sir" + julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" check "$file" + out="$(julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" run "$file" --trace)" + echo "$out" + counted[$o]="$(sed -n 's/.*peak_live=\([0-9]*\).*/\1/p' <<<"$out" | head -1)" +done + +echo "== Typed peak (Agda, HWM Timeline)" +check_agda docs/occupancy/TypedPeak.agda TypedPeak.agda + +# The graded reading (fork issue #105): occupancy-types' Reclaim theorems at the +# measured grades, from the same pinned siblings. The recorded run's grades +# against its measurement are checked by scripts/check-occupancy-grades.jl, which +# needs the project's Julia packages and so runs in the disagreement job. +check_agda docs/occupancy/GradedStages.agda GradedStages.agda +for file in docs/occupancy/reject/*.agda; do + name="$(basename -- "$file" .agda)" + log="$logs/$name.log" + if agda "${flags[@]}" -i docs/occupancy/reject "$file" >"$log" 2>&1; then + cat "$log" + annotate "Invalid claim accepted: $name" "$file" "$log" + echo "ERROR: invalid claim accepted: $name" >&2; failures=$((failures + 1)); continue + fi + if ! grep -Eq "/$name\.agda:[0-9]+,[0-9]+" "$log" || ! grep -q 'when checking that' "$log" || + grep -Eq 'Not in scope|Failed to find|Unsolved|Parse error' "$log"; then + cat "$log" + annotate "$name failed for the wrong reason" "$file" "$log" + echo "ERROR: $name failed for the wrong reason" >&2; failures=$((failures + 1)); continue + fi + echo "PASS: rejected $name ($(grep -m1 -E '[0-9]+ != [0-9]+' "$log"))" +done + +echo "== Counted bytes against typed peak (KiB)" +printf '%-14s %10s %10s\n' order counted typed +for o in "${orders[@]}"; do + typed="$(grep -A1 -E "^$o :" docs/occupancy/TypedPeak.agda | sed -n 's/.*≡ \([0-9]*\)$/\1/p' | head -1)" + printf '%-14s %10s %10s\n' "$o" "${counted[$o]}" "$typed" + if [ -z "${counted[$o]}" ] || [ -z "$typed" ] || [ "${counted[$o]}" != "$typed" ]; then + echo "ERROR: $o: counted ${counted[$o]:-?} != typed ${typed:-?}" >&2; failures=$((failures + 1)) + echo "counted ${counted[$o]:-?} KiB != typed ${typed:-?} KiB" >"$logs/counted-$o.log" + annotate "Counted bytes != typed peak ($o)" docs/occupancy/TypedPeak.agda "$logs/counted-$o.log" + fi +done + +if (( failures > 0 )); then + echo "check-occupancy: $failures failure(s)" >&2 + exit 1 +fi +echo "check-occupancy: OK" diff --git a/scripts/fetch-siblings.sh b/scripts/fetch-siblings.sh new file mode 100755 index 00000000..31a98bc1 --- /dev/null +++ b/scripts/fetch-siblings.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fetch the proof siblings the occupancy reading (docs/occupancy-2026-10.adoc) +# consumes, each at its pinned commit, into _build/dependencies. Their +# definitions are used, never copied. Re-pin one sibling per commit. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +# occupancy-types: the HWM Timeline (Agda) and the SessionIR stepper (Julia). +OCCUPANCY_TYPES_COMMIT="${OCCUPANCY_TYPES_COMMIT:-af957280075699c659de0abd9a3b0defade77309}" +# resource-algebra: the HWM grade algebra occupancy-types builds on. +RESOURCE_ALGEBRA_COMMIT="${RESOURCE_ALGEBRA_COMMIT:-cf9235c532712a783a404c359a1055f40134794a}" + +fetch() { + local name="$1" commit="$2" dir="_build/dependencies/$1" + if [ ! -d "$dir/.git" ]; then + git init --quiet "$dir" + git -C "$dir" remote add origin "https://github.com/hyperpolymath/$name.git" + fi + git -C "$dir" fetch --quiet --depth 1 origin "$commit" + git -C "$dir" checkout --quiet --detach "$commit" + echo "$name at $(git -C "$dir" rev-parse --short HEAD)" +} + +mkdir -p _build/dependencies +fetch occupancy-types "$OCCUPANCY_TYPES_COMMIT" +fetch resource-algebra "$RESOURCE_ALGEBRA_COMMIT" diff --git a/install.jl b/scripts/install.jl similarity index 99% rename from install.jl rename to scripts/install.jl index 7e835e62..f3465b3b 100644 --- a/install.jl +++ b/scripts/install.jl @@ -6,7 +6,7 @@ # required R packages. Writes resolved tool paths to config/tools.yml. # # Usage via install.sh, or directly by: -# julia --project=. install.jl [--update] [--modify] [--sysimage] +# julia --project=. scripts/install.jl [--update] [--modify] [--sysimage] # # Options: # --update Re-assert the pinned tool versions and refresh managed binaries in bin/ @@ -41,7 +41,7 @@ import Downloads const UPDATE_MODE = "--update" in ARGS const MODIFY_MODE = "--modify" in ARGS const SYSIMAGE_MODE = "--sysimage" in ARGS -const PROJECT_ROOT = @__DIR__ +const PROJECT_ROOT = normpath(dirname(@__DIR__)) const BIN_DIR = joinpath(PROJECT_ROOT, "bin") const CONFIG_DIR = joinpath(PROJECT_ROOT, "config") const TOOLS_CONFIG = joinpath(CONFIG_DIR, "tools.yml") @@ -1025,7 +1025,7 @@ end ## Sysimage creation const SYSIMAGE_EXT = Sys.isapple() ? ".dylib" : ".so" const SYSIMAGE_PATH = joinpath(PROJECT_ROOT, "MetaManifold$(SYSIMAGE_EXT)") -const PRECOMPILE_EXEC_PATH = joinpath(PROJECT_ROOT, "precompile_exec.jl") +const PRECOMPILE_EXEC_PATH = joinpath(@__DIR__, "precompile_exec.jl") function build_sysimage() # PackageCompiler is a project dependency, installed by Pkg.instantiate(). @@ -1247,7 +1247,7 @@ function main() record!("Julia sysimage", STEP_OK, SYSIMAGE_PATH) catch e @error "Sysimage build failed: $e" - record!("Julia sysimage", STEP_FAILED, "rerun: julia --project=. install.jl --sysimage") + record!("Julia sysimage", STEP_FAILED, "rerun: julia --project=. scripts/install.jl --sysimage") end else record!("Julia sysimage", STEP_OK, "not requested (pass --sysimage to build one)") diff --git a/scripts/kyaml-format.sh b/scripts/kyaml-format.sh new file mode 100755 index 00000000..145b157d --- /dev/null +++ b/scripts/kyaml-format.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# kyaml-format.sh — format arbitrary YAML as KYAML, or check that it already is +# (YAML-POLICY Y-3; KYAML adoption step 3, standards#1022). +# +# The rewriter is `yq -p yaml -o kyaml '.'`, verbatim — the one rewriter the +# step-2 comment-preservation proof (tools/yaml-comment-proof, standards#1021) +# covers. That proof does not generalise to any other rewriter, so this script +# deliberately adds none: no post-processing, no re-indenting, no sed. +# +# Usage: +# kyaml-format.sh [--check] FILE... +# +# (default) rewrite each FILE in place as KYAML when it is not already. +# --check rewrite nothing; exit 1 if any FILE is not already KYAML. +# +# Exit contract: +# 0 = every FILE is KYAML (after the rewrite, in format mode) +# 1 = --check: at least one FILE is not KYAML (each one is named) +# 2 = refused: no FILE given, a FILE is missing or unparseable, a FILE has no +# final newline, or the rewrite would change the parsed data. A refusal +# never rewrites that FILE and never counts as clean. +# +# Why the final-newline refusal: a `|` block scalar that ends at EOF with no +# line break is read differently by go-yaml (yq) and eemeli/yaml — measured on +# k9-contractile.yml, the trailing "\n" of its last run: body exists for one +# parser and not the other. Formatting such a file would silently pick one +# reading, so the script refuses it and names the fix instead. +set -uo pipefail + +MODE=format +if [ "${1:-}" = "--check" ]; then + MODE=check + shift +fi + +if [ "$#" -eq 0 ]; then + echo "kyaml-format: no files given (usage: kyaml-format.sh [--check] FILE...)" >&2 + exit 2 +fi + +command -v yq >/dev/null 2>&1 || { echo "kyaml-format: yq is not on PATH" >&2; exit 2; } + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# canonical_data — prints the file's parsed value as key-sorted JSON; +# fails when the file does not parse. +canonical_data() { + yq -p yaml -o json -I 0 'sort_keys(..)' "$1" +} + +# kyaml_of — writes the KYAML rendering of to ; +# fails when does not parse. +kyaml_of() { + yq -p yaml -o kyaml '.' "$1" > "$2" +} + +# refuse — reports a file this script will not judge. +refuse() { + echo "::error file=$1::kyaml-format: REFUSED $1 — $2" >&2 + refused=$((refused + 1)) +} + +refused=0 drift=0 clean=0 rewritten=0 +i=0 +for f in "$@"; do + i=$((i + 1)) + out="$TMP/$i.kyaml" + if [ ! -f "$f" ]; then + refuse "$f" "no such file"; continue + fi + if [ -s "$f" ] && [ -n "$(tail -c1 "$f")" ]; then + refuse "$f" "no final newline; add one first (a trailing block scalar is ambiguous without it)"; continue + fi + if ! kyaml_of "$f" "$out" 2>"$TMP/err"; then + refuse "$f" "does not parse: $(head -1 "$TMP/err")"; continue + fi + if cmp -s "$f" "$out"; then + clean=$((clean + 1)); continue + fi + # The rewrite is only acceptable when it preserves the data exactly. + if ! before=$(canonical_data "$f") || ! after=$(canonical_data "$out") || [ "$before" != "$after" ]; then + refuse "$f" "the KYAML rendering does not parse to the same data"; continue + fi + if [ "$MODE" = check ]; then + echo "::error file=$f::kyaml-format: $f is not KYAML — run scripts/kyaml-format.sh $f" >&2 + drift=$((drift + 1)) + else + cat "$out" > "$f" + echo "kyaml-format: rewrote $f" + rewritten=$((rewritten + 1)) + fi +done + +total=$# +if [ "$refused" -gt 0 ]; then + echo "kyaml-format: $refused of $total file(s) REFUSED — not judged, not clean" >&2 + exit 2 +fi +if [ "$drift" -gt 0 ]; then + echo "kyaml-format: $drift of $total file(s) are not KYAML" >&2 + exit 1 +fi +if [ "$MODE" = check ]; then + echo "kyaml-format: all $total file(s) are KYAML" +else + echo "kyaml-format: $total file(s) KYAML ($rewritten rewritten, $clean already clean)" +fi +exit 0 diff --git a/scripts/lint-occupancy-readings.sh b/scripts/lint-occupancy-readings.sh new file mode 100755 index 00000000..114779a0 --- /dev/null +++ b/scripts/lint-occupancy-readings.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fork issue #105: a task's recorded `peak_rss_kib` is a reading, not an +# occupancy. What a stage occupied is determined only by what the next stage +# inherits from it — the drift — which is why the probe grades a whole window and +# why src/core/memory_probe.jl's helpers (`graded`, `predicted_peak_rss_kib`, +# `cost_bound_rss_kib`, `reclaimed_kib`, `task_grades`) exist. Code outside the +# probe that reaches for the key itself is comparing measured figures without +# that context; on the recorded run that reading is off by 9180 KiB for +# `asv_then_otu` and 9204 KiB for `otu_then_asv` (the reclaimed suffix). +# +# So: production code may name the key only in src/core/memory_probe.jl, where +# the record is built and graded. Tests may read it (they assert the recorded +# numbers directly, which is fine — they are evidence, not presentation). +# +# Run from the repository root: bash scripts/lint-occupancy-readings.sh +set -u -o pipefail + +if [[ ! -f src/core/memory_probe.jl ]]; then + echo "lint-occupancy-readings: run from the repository root" >&2 + exit 2 +fi + +failures=0 +while IFS= read -r hit; do + [[ -z "$hit" ]] && continue + echo "ERROR: ungraded occupancy reading: $hit" >&2 + failures=$((failures + 1)) +done < <(grep -rn --include='*.jl' '"peak_rss_kib"' src/ | + grep -v '^src/core/memory_probe\.jl:' || true) + +if (( failures > 0 )); then + echo "lint-occupancy-readings: $failures ungraded reading(s); use the grading helpers in" >&2 + echo "MemoryProbe (see docs/occupancy-2026-10.adoc, 'Reading the numbers') instead." >&2 + exit 1 +fi + +echo "lint-occupancy-readings: OK — every peak_rss_kib reading in src/ goes through the probe's grades" diff --git a/precompile_exec.jl b/scripts/precompile_exec.jl similarity index 100% rename from precompile_exec.jl rename to scripts/precompile_exec.jl diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index 31db2a5c..b5f13a59 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -33,6 +33,9 @@ include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/differential.jl") +# Canonical JSON: deterministic bytes for hashing and content-addressing. +include("canonical.jl") + # Web server include("server/server.jl") diff --git a/src/analysis/analysis.jl b/src/analysis/analysis.jl index f40bcc2e..2b0105d2 100644 --- a/src/analysis/analysis.jl +++ b/src/analysis/analysis.jl @@ -950,14 +950,25 @@ _not_computed(status::Symbol, reason::AbstractString; """ was_computed(r::AlphaSignificance) -> Bool -Whether `r` holds an actual test result. False means no test was performed, so +Whether `r` has status `:computed`. False means no omnibus result is available, so neither `omnibus` nor `pairwise` may be presented as a finding. """ was_computed(r::AlphaSignificance) = r.status === :computed """ Run the omnibus (and optionally pairwise) alpha-diversity test, returning an -`AlphaSignificance` whose status says whether a test was actually performed. +`AlphaSignificance` whose status distinguishes a computed omnibus result from +an unavailable or failed test. + +`values`, group `labels` and `sample_ids` describe the same samples. Unpaired +data use Kruskal-Wallis; `paired_samples` uses common sample IDs across all +groups, averaging repeated IDs within a group, with paired Wilcoxon for two +groups or Friedman for more. `pairwise` requests Wilcoxon comparisons with +Benjamini-Hochberg adjusted p-values. + +Unavailable R/vegan, no common IDs for pairing, or an omnibus R error converted +to a missing p-value return statuses `:r_unavailable`, `:no_paired_samples` or +`:test_failed`, respectively. A failed omnibus result retains any pairwise rows. Waits for the R runtime like the other R analyses. While a pipeline run holds it, `RBusyError` reaches the server, which reports the chart as unavailable @@ -1163,6 +1174,22 @@ function _add_pairwise_annotations!(layout::Dict{String,Any}, layout[yaxis_layout_key] = axis end +""" + alpha_boxplot(groups; show_points=true, annotate_significance=false, + pairwise_brackets=false, paired_samples=false, + paired_lines=false, significance_test="kruskal_wallis") -> Dict + +Build a Plotly figure with richness, Shannon and Gini-Simpson boxplot panels. +Each group is `(label, sample_ids, richness, shannon, simpson)`, with metric +values aligned to its sample IDs. `paired_lines` joins matching IDs across +groups and disables point jitter. `paired_samples` selects paired tests when +an ID is shared by every group; otherwise tests are unpaired. + +With at least two group labels and `significance_test == "kruskal_wallis"`, +tests run even when neither annotation option is enabled. The options display +omnibus captions or pairwise brackets, including notices for unavailable or +failed tests. `RBusyError` propagates when the R runtime stays busy. +""" function alpha_boxplot(groups::AbstractVector{<:Tuple{String, Vector{String}, AbstractVector{<:Real}, Vector{Float64}, Vector{Float64}}}; show_points::Bool=true, annotate_significance::Bool=false, @@ -1442,11 +1469,13 @@ end PERMANOVA via vegan::adonis2 with up to 999 permutations. `metadata` is a DataFrame with one row per sample and covariate columns. -Each covariate is tested as its own term (`by = "terms"`), sequentially in +Columns named `sample` (case-insensitive) and constant columns are excluded. +Each remaining covariate is tested as its own term (`by = "terms"`), sequentially in column order, so each row is conditioned on the terms before it. The top-level `term`, `r2`, `f_statistic` and `p_value` are those of the first term. `blocks`, one label per sample (e.g. the individual a sample came from), -restricts permutations to within blocks. It is ignored when no label repeats, +restricts permutations to within blocks. It is ignored when its length does +not match the sample count or when no label repeats, since singleton blocks admit no permutation at all. A term that is constant within every block cannot be tested by within-block permutation (each sample never leaves its block, so the permutations never exchange that term's labels @@ -1456,6 +1485,12 @@ and the p-value does not test it), so its p-value is withheld and than 999 when the blocks admit fewer, and `min_p_value` the smallest p-value that number can produce. `transform` is applied before `vegdist`; see [`run_nmds`](@ref). + +Returns `nothing` when R/vegan is unavailable or no covariates remain. An +`adonis2` error becomes a named tuple containing `message`; on success, +`dispersion` contains PERMDISP results for the combined covariate groups or +an `error` message. `RBusyError`, transformation errors and R errors outside +these caught tests propagate. """ function run_permanova(mat::Matrix{Float64}, metadata::DataFrame; seed::Integer=123, transform::String="none", diff --git a/src/analysis/differential.jl b/src/analysis/differential.jl index d2c78de0..864bd2b7 100644 --- a/src/analysis/differential.jl +++ b/src/analysis/differential.jl @@ -22,13 +22,18 @@ using DataFrames, RCall using ..RRuntime: with_r_lock using ..Analysis: _palette_hex, R_WAIT_SECONDS export DifferentialConfig, ScalingRefusal, MASSUnavailable, ZCompositionsUnavailable, - OFFSET_METHODS, METHODS, tss_factors, rle_factors, size_factors, bh_adjust, - validate_counts, replace_zeros, clr_transform, fit_nb, fit_welch, - differential_abundance, volcano_chart + GlmGamPoiUnavailable, OFFSET_METHODS, METHODS, tss_factors, rle_factors, + size_factors, bh_adjust, validate_counts, replace_zeros, clr_transform, + fit_nb, fit_welch, differential_abundance, volcano_chart # Methods accepted under `analysis.differential.method`. const METHODS = ("nb_glm", "clr_welch") +# The per-taxon dispersion `nb_glm` uses: maximum likelihood (MASS::glm.nb's +# own theta) or glmGamPoi's shrinkage estimator, which shares information +# across taxa. `"ml"` is the default and keeps #24's results unchanged. +const DISPERSIONS = ("ml", "glmGamPoi") + # Offset (size factor) methods accepted under `analysis.differential.offset`. const OFFSET_METHODS = ("tss", "rle") @@ -43,7 +48,7 @@ const DEFAULT_DELTA = 0.65 """ DifferentialConfig(offset="tss", min_prevalence=0.0; method="nb_glm", - replacement_delta=0.65) + replacement_delta=0.65, dispersion="ml") Settings for one differential abundance analysis, read from `analysis.differential` in the pipeline config. `method` is `"nb_glm"` or @@ -54,26 +59,45 @@ log-ratios, which are already free of library size. have at least one read to be tested. `replacement_delta`, in (0, 1), is the `frac` passed to `zCompositions::cmultRepl` by `clr_welch`: an imputed proportion above its taxon's smallest observed proportion is lowered to this fraction of it. +`dispersion` names the per-taxon dispersion `nb_glm` uses (`"ml"` or +`"glmGamPoi"`): `"ml"` takes the maximum-likelihood theta from `MASS::glm.nb`, +`"glmGamPoi"` takes the shrunken dispersion from `glmGamPoi::glm_gp` -- fitted +across all taxa in one call, which is where its shared information helps low-count +taxa -- and fits each taxon through `MASS::negative.binomial(theta = ...)` so the +Wald test and its checks are unchanged. It is `nb_glm` only. +Method and offset names are stripped and lowercased. Throws `ArgumentError` +for unsupported names or out-of-range fractions, including settings unused by +the selected method. """ struct DifferentialConfig method :: String offset :: String min_prevalence :: Float64 replacement_delta :: Float64 + dispersion :: String function DifferentialConfig(offset::AbstractString="tss", min_prevalence::Real=0.0; method::AbstractString="nb_glm", - replacement_delta::Real=DEFAULT_DELTA) + replacement_delta::Real=DEFAULT_DELTA, + dispersion::AbstractString="ml") m = lowercase(strip(String(method))) m in METHODS || throw(ArgumentError( "analysis.differential.method must be one of $(join(METHODS, ", ")), not '$method'")) o = lowercase(strip(String(offset))) o in OFFSET_METHODS || throw(ArgumentError( "analysis.differential.offset must be one of $(join(OFFSET_METHODS, ", ")), not '$offset'")) + d = strip(String(dispersion)) + di = findfirst(s -> lowercase(s) == lowercase(d), DISPERSIONS) + di === nothing && throw(ArgumentError( + "analysis.differential.dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) + d = DISPERSIONS[di] + (d == "ml" || m == "nb_glm") || throw(ArgumentError( + "analysis.differential.dispersion is a negative-binomial setting, but method is " * + "'$m'; only nb_glm has a dispersion to choose")) (isfinite(min_prevalence) && 0 <= min_prevalence <= 1) || throw(ArgumentError( "analysis.differential.min_prevalence must be a fraction in [0, 1], not $min_prevalence")) (isfinite(replacement_delta) && 0 < replacement_delta < 1) || throw(ArgumentError( "analysis.differential.replacement_delta must be in (0, 1), not $replacement_delta")) - new(m, o, Float64(min_prevalence), Float64(replacement_delta)) + new(m, o, Float64(min_prevalence), Float64(replacement_delta), d) end end @@ -115,6 +139,18 @@ Base.showerror(io::IO, ::ZCompositionsUnavailable) = print(io, "the R package zCompositions is not available, so zeros cannot be replaced for " * "clr_welch; restore the R library from renv.lock (Rscript -e 'renv::restore()')") +""" + GlmGamPoiUnavailable() + +Raised when the R package glmGamPoi, which provides the shrunken dispersion +`analysis.differential.dispersion: glmGamPoi` asks for, cannot be loaded. +""" +struct GlmGamPoiUnavailable <: Exception end + +Base.showerror(io::IO, ::GlmGamPoiUnavailable) = print(io, + "the R package glmGamPoi is not available, so the shrunken dispersion cannot be " * + "fitted; restore the R library from renv.lock (Rscript -e 'renv::restore()')") + """ _geomean(x) -> Float64 @@ -243,7 +279,36 @@ da_result <- data.frame(status = rep("ok", da_n), note = rep("", da_n), estimate = rep(NA_real_, da_n), se = rep(NA_real_, da_n), statistic = rep(NA_real_, da_n), pvalue = rep(NA_real_, da_n), theta = rep(NA_real_, da_n), stringsAsFactors = FALSE) +# glmGamPoi's shrunken dispersion, fitted across all taxa in one call -- its +# genes x samples convention, so the counts are transposed, and size_factors +# is the linear size factor (the per-taxon fits below take its log as the +# offset). A fit error or a dispersion that is not one finite positive value +# per taxon fails every taxon with the reason: with theta given, not +# estimated, there is nothing left to fall back to. +da_phi <- rep(NA_real_, da_n) +da_gp_failed <- "" +if (da_use_gp) { + da_gp <- tryCatch( + glmGamPoi::glm_gp(t(da_counts), design = model.matrix(~ da_g), + size_factors = exp(da_offset)), + error = function(e) e) + if (inherits(da_gp, "error")) { + da_gp_failed <- paste("glm_gp stopped with an error:", conditionMessage(da_gp)) + } else { + da_phi <- as.numeric(da_gp[["overdispersions"]]) + if (length(da_phi) != da_n || any(!is.finite(da_phi)) || any(da_phi <= 0)) { + da_gp_failed <- paste("glm_gp returned a dispersion that is not one finite", + "positive value per taxon") + da_phi <- rep(NA_real_, da_n) + } + } +} for (da_j in seq_len(da_n)) { + if (nzchar(da_gp_failed)) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- da_gp_failed + next + } da_y <- da_counts[, da_j] if (length(unique(da_y)) < 2L) { da_result[da_j, "status"] <- "failed" @@ -263,18 +328,48 @@ for (da_j in seq_len(da_n)) { next } da_warns <- character(0) - da_fit <- tryCatch( - withCallingHandlers( - MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), - warning = function(w) { - da_warns <<- c(da_warns, conditionMessage(w)) - invokeRestart("muffleWarning") - }), - error = function(e) e) - if (inherits(da_fit, "error")) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) - next + # "ml": theta is estimated per taxon by glm.nb's own alternating iterations. + # "glmGamPoi": theta is TAKEN from the shrunken dispersion fitted above + # (theta = 1 / phi) and the per-taxon fit goes through glm() with MASS's + # negative.binomial family, so the Wald test and every check below are the + # same machinery on a given dispersion instead of an estimated one. + if (da_use_gp) { + da_theta <- 1 / da_phi[da_j] + if (!is.finite(da_theta) || da_theta <= 0) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- "the shrunken dispersion for this taxon is not usable" + next + } + da_fit <- tryCatch( + withCallingHandlers( + glm(da_y ~ da_g + offset(da_offset), + family = MASS::negative.binomial(theta = da_theta), + control = glm.control(maxit = 100)), + warning = function(w) { + da_warns <<- c(da_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + }), + error = function(e) e) + if (inherits(da_fit, "error")) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- paste("the negative-binomial fit stopped with an error:", + conditionMessage(da_fit)) + next + } + } else { + da_fit <- tryCatch( + withCallingHandlers( + MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), + warning = function(w) { + da_warns <<- c(da_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + }), + error = function(e) e) + if (inherits(da_fit, "error")) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) + next + } } da_warns <- unique(da_warns) if (!isTRUE(da_fit[["converged"]]) || !is.null(da_fit[["th.warn"]]) || @@ -296,15 +391,23 @@ for (da_j in seq_len(da_n)) { da_result[da_j, "note"] <- "the fit returned a non-finite estimate, standard error or p-value" next } - da_theta <- da_fit[["theta"]] + # "glmGamPoi" set da_theta before the fit; "ml" takes it from the fit. + if (!da_use_gp) da_theta <- da_fit[["theta"]] da_result[da_j, c("estimate", "se", "statistic", "pvalue")] <- unname(da_row[1:4]) da_result[da_j, "theta"] <- da_theta if (!is.finite(da_theta) || da_theta >= da_theta_upper) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" + da_result[da_j, "note"] <- if (da_use_gp) + paste("the shrunken dispersion puts these counts at the Poisson end:", + "no overdispersion is left, so the fit is effectively Poisson") + else + "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" } else if (da_theta <= da_theta_lower) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" + da_result[da_j, "note"] <- if (da_use_gp) + "the shrunken dispersion puts these counts at the extreme-variance end" + else + "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" } else if (length(da_warns) > 0L) { da_result[da_j, "note"] <- paste("R warned:", paste(da_warns, collapse = "; ")) } @@ -319,36 +422,50 @@ A finite number as `Float64`, or `nothing` for R's NA, NaN or an infinity. _num(x) = (ismissing(x) || isnothing(x) || !isfinite(x)) ? nothing : Float64(x) """ - fit_nb(counts, groups, offset; levels) -> DataFrame + fit_nb(counts, groups, offset; levels, dispersion="ml") -> DataFrame Fit `MASS::glm.nb(y ~ group + offset(offset))` to each column of the samples x taxa integer matrix `counts`. `levels` is `(reference, contrast)`, so the estimate is the natural-log fold change of `contrast` over `reference`. +`dispersion` names where each taxon's theta comes from: `"ml"` (the default) +takes `glm.nb`'s own maximum-likelihood estimate, `"glmGamPoi"` takes the +shrunken dispersion `glmGamPoi::glm_gp` fits across all taxa in one call and +runs each per-taxon fit through `MASS::negative.binomial(theta = 1/phi)`, so +the Wald test and its checks are unchanged. Returns one row per taxon with `status` (`ok`, `boundary` or `failed`), `note`, `estimate`, `se`, `statistic`, `pvalue` and `theta`; a failed taxon has `nothing` for every statistic. Throws `MASSUnavailable` when MASS cannot be -loaded and `RBusyError` when the R runtime stays busy. +loaded, `GlmGamPoiUnavailable` when `dispersion` is `"glmGamPoi"` and glmGamPoi +cannot be loaded, and `RBusyError` when the R runtime stays busy. """ function fit_nb(counts::AbstractMatrix{<:Integer}, groups::AbstractVector{<:AbstractString}, - offset::AbstractVector{<:Real}; levels::NTuple{2,String}) + offset::AbstractVector{<:Real}; levels::NTuple{2,String}, + dispersion::AbstractString="ml") + d = strip(String(dispersion)) + lowercase(d) in lowercase.(DISPERSIONS) || throw(ArgumentError( + "dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) + use_gp = lowercase(d) == "glmgampoi" size(counts, 1) == length(groups) == length(offset) || throw(ArgumentError( "counts have $(size(counts, 1)) samples, groups $(length(groups)), offset $(length(offset))")) raw = with_r_lock(; timeout=R_WAIT_SECONDS[]) do RCall.rcopy(RCall.reval("requireNamespace('MASS', quietly = TRUE)")) || throw(MASSUnavailable()) + use_gp && !RCall.rcopy(RCall.reval("requireNamespace('glmGamPoi', quietly = TRUE)")) && + throw(GlmGamPoiUnavailable()) RCall.globalEnv[:da_counts] = Matrix{Int}(counts) RCall.globalEnv[:da_group] = String.(groups) RCall.globalEnv[:da_levels] = collect(levels) RCall.globalEnv[:da_offset] = Float64.(offset) RCall.globalEnv[:da_theta_upper] = THETA_UPPER RCall.globalEnv[:da_theta_lower] = THETA_LOWER + RCall.globalEnv[:da_use_gp] = use_gp try RCall.reval(_FIT_R) DataFrame(RCall.rcopy(RCall.reval("da_result"))) finally RCall.reval("rm(list = intersect(ls(), c('da_counts', 'da_group', 'da_levels', " * - "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_g', 'da_term', " * - "'da_n', 'da_result', 'da_j', 'da_y', 'da_warns', 'da_fit', 'da_co', " * - "'da_row', 'da_theta', 'da_absent'))); invisible(gc())") + "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_use_gp', 'da_g', 'da_term', " * + "'da_n', 'da_result', 'da_phi', 'da_gp_failed', 'da_gp', 'da_j', 'da_y', " * + "'da_warns', 'da_fit', 'da_co', 'da_row', 'da_theta', 'da_absent'))); invisible(gc())") end end DataFrame(status = String.(raw.status), @@ -365,7 +482,8 @@ end Centred log-ratio of each row (sample) of the samples x taxa matrix `x`: `log(x_ij)` minus the mean of `log(x_i·)`, so every row sums to zero. Every -value must be positive; replace zeros first. +value must be finite and positive or an `ArgumentError` is thrown; replace +zeros first. """ function clr_transform(x::AbstractMatrix{<:Real}) all(v -> isfinite(v) && v > 0, x) || throw(ArgumentError( @@ -398,8 +516,9 @@ da_out <- tryCatch( Check that cmultRepl's output `out` is one valid composition per sample: the same shape as the input whose zeros `zero_mask` marks, and every row finite, -positive and summing to 1. Throws `ArgumentError` naming the first sample that -fails, pointing at `min_prevalence`. +positive and summing to 1 within an absolute tolerance of 1e-8. Throws +`ErrorException` if the shapes differ, or `ArgumentError` naming the first +invalid sample and pointing at `min_prevalence`. Returns `out` unchanged. """ function _check_compositions(out::AbstractMatrix{<:Real}, zero_mask::AbstractMatrix{Bool}, samples::AbstractVector{<:AbstractString}) @@ -427,6 +546,9 @@ data. Returns `(proportions, zeros_replaced, imputed_mass, warnings)`: `proportions` has the same shape as `counts`, each row summing to 1 with no zero left; `imputed_mass` is the share of each sample's total held by replaced values; `warnings` is whatever R warned. +`frac` controls the fraction of a taxon's smallest observed proportion used +to lower an excessive imputed proportion (see [`DifferentialConfig`](@ref)). +Without zeros, returns row-normalised counts, but still requires zCompositions. The caller chooses the taxa: everything passed in is replaced and kept, none is dropped. Throws `ArgumentError` when there are fewer than 2 taxa, when a taxon @@ -434,7 +556,8 @@ has reads in fewer than 2 samples (the GBM prior is then undefined), when a sample has no reads, or when cmultRepl refuses or returns a value that is not a positive proportion; the messages name the taxon or sample and point at `min_prevalence`. Throws `ZCompositionsUnavailable` when zCompositions cannot -be loaded and `RBusyError` when the R runtime stays busy. +be loaded and `RBusyError` when the R runtime stays busy. Mismatched sample or +taxon labels raise `ArgumentError`; a changed output shape raises `ErrorException`. """ function replace_zeros(counts::AbstractMatrix{<:Real}; frac::Real=DEFAULT_DELTA, @@ -540,7 +663,10 @@ the mean of `contrast` minus the mean of `reference`; the variances of the two groups are not assumed equal, since the groups often differ in size. Returns one row per taxon with `status` (`ok` or `failed`), `note`, `estimate`, `se`, `statistic` (Welch's t) and `pvalue`; a failed taxon has `nothing` for every -statistic. Throws `RBusyError` when the R runtime stays busy. +statistic. Errors from an individual R test become failed rows with a reason +in `note`. Throws `ArgumentError` if the group labels do not match the number +of samples and `RBusyError` when the R runtime stays busy; other R evaluation +or conversion errors propagate. """ function fit_welch(values::AbstractMatrix{<:Real}, groups::AbstractVector{<:AbstractString}; levels::NTuple{2,String}) @@ -601,19 +727,23 @@ every observed taxon first would let imputed values for rare taxa take over the samples of a sparse table. The estimate is the difference in mean CLR, contrast minus reference: a difference in centred-log-ratio units, not a fold change, relative to the geometric mean of the tested taxa. Replaced zeros are -not measurements; `diagnostics.zero_replacement` reports their share of each -sample. Each group needs at least 2 samples. +not measurements; `diagnostics.zero_replacement` reports the median and maximum +share of sample mass held by replaced zeros. Each group needs at least 2 samples. For both methods, taxa present in fewer than `config.min_prevalence` of the samples are reported as `filtered` and not fitted, and the Benjamini-Hochberg family is the fitted taxa that produced a p-value; failed and filtered taxa carry `padj = nothing`. The result's `effect` names the row key that holds the effect size and its label. +Rows are sorted by adjusted p-value, raw p-value and taxon name, with missing +p-values last. The overall status is `partial` if any taxon failed, otherwise `ok`. Throws `ArgumentError` for unusable input (including a zero replacement that cannot be made), `ScalingRefusal` when the offset cannot be computed, `MASSUnavailable` or `ZCompositionsUnavailable` when the R package a method -needs is missing, and an `ErrorException` when no taxon could be fitted at all. +needs is missing, `RBusyError` when the R runtime stays busy, and an +`ErrorException` when no taxon could be fitted at all or zero replacement changes +the table's shape. Other R evaluation or conversion errors propagate. """ function differential_abundance(counts::AbstractMatrix{<:Real}, samples::AbstractVector{<:AbstractString}, @@ -651,7 +781,8 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, factors = size_factors(counts, samples, config.offset) tested = findall(>=(config.min_prevalence), prevalence) fits = isempty(tested) ? nothing : - fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels) + fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels, + dispersion=config.dispersion) if !isnothing(fits) for (k, j) in enumerate(tested) f = fits[k, :] @@ -669,7 +800,8 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, size_factor_rows = [Dict("sample" => String(samples[i]), "group" => String(groups[i]), "factor" => factors[i]) for i in eachindex(samples)] config_echo = Dict{String,Any}("method" => config.method, "offset" => config.offset, - "min_prevalence" => config.min_prevalence) + "min_prevalence" => config.min_prevalence, + "dispersion" => config.dispersion) else (n_ref >= 2 && n_con >= 2) || throw(ArgumentError( "Welch's t-test needs at least 2 samples in each group to estimate its variance; " * @@ -750,8 +882,9 @@ end A Plotly volcano plot of a `differential_abundance` result: the effect named by `result["effect"]` (log2 fold change, or CLR difference) against -log10 of the raw p-value, coloured by whether the BH-adjusted p-value is below `alpha`. Taxa -without a p-value are omitted here; they remain in the results table with their -reason. +without an adjusted p-value are omitted here; they remain in the results table +with their reason. Without `effect`, uses log2 fold change. Zero raw p-values +are plotted at the smallest positive normal `Float64` and identified in hover text. """ function volcano_chart(result::AbstractDict; alpha::Real=0.05) ref = result["groups"]["reference"] diff --git a/src/canonical.jl b/src/canonical.jl new file mode 100644 index 00000000..04363037 --- /dev/null +++ b/src/canonical.jl @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# JCS (RFC 8785) canonical JSON: deterministic bytes for hashing, signing and +# content-addressing. The same value always yields the same byte string, so a +# chart spec or an analysis result can be hashed where it is produced and the +# hash compared across machines, Julia versions and refactorings that build +# the same object in a different order. +# +# What this implements, and what it deliberately does not: +# +# * Object keys are sorted by UTF-16 code unit, the rule that makes an +# astral-plane key sort BEFORE a key such as U+FF21, because its high +# surrogate (U+D800..U+DBFF) is a smaller code unit than U+FF21. +# * Float64 values serialize per ECMAScript's Number::toString, the rules +# RFC 8785 builds on: shortest round-trip digits, positional notation for +# 1e-6 <= |x| < 1e21, exponent notation outside it, a signed exponent +# ("1e+30"), and -0.0 collapsing to 0. +# * Integers outside I-JSON's exact range (-(2^53)+1 .. 2^53-1) are +# refused, as the estate's ijson-jcs crate refuses them, because a +# consumer cannot read them back as the same number. +# * NaN and Inf are refused for the same reason. +# +# The implementation is verified against the estate's ijson-jcs crate +# (Rust, RFC 7493 + RFC 8785): the fixtures in +# test/fixtures/jcs_oracle.jsonl were produced BY that crate, and +# test/unit/test_canonical.jl asserts this module reproduces them +# byte-for-byte. The crate is the oracle; this is the second +# implementation, and any disagreement between them is a bug in one of +# them. + +module Canonical + + +using SHA + +export jcs_bytes, jcs_sha256 + +# The I-JSON exact-integer range. +const SAFE_INT_MAX = 2^53 - 1 + +""" + jcs_bytes(x) -> Vector{UInt8} + +The RFC 8785 canonical form of `x`, as bytes. Accepts the JSON-shaped +values the server produces: `Dict` and `JSON3.Object` (key order is +irrelevant; it is sorted away), `AbstractVector` and `JSON3.Array` (order +is preserved, as JCS preserves array order), `AbstractString`, `Integer`, +`AbstractFloat`, `Bool` and `nothing`. Floats serialize per ECMAScript's +Number::toString; integers outside the exact I-JSON range, and NaN or Inf +floats, throw `ArgumentError`. +""" +function jcs_bytes(x) + io = IOBuffer() + _write!(io, x) + take!(io) +end + +""" + jcs_sha256(x) -> String + +The SHA-256 of `jcs_bytes(x)`, as lowercase hex -- the content address of +a JSON value: two producers that build the same value hash identically +whatever order they built it in. +""" +jcs_sha256(x) = bytes2hex(sha256(jcs_bytes(x))) + +# Object keys sort by UTF-16 code unit, which for astral-plane characters +# means their surrogate pair, a smaller first unit than any BMP code point +# at or above U+E000. transcoding the key to UInt16 gives exactly those +# units to compare. +_utf16_units(s::AbstractString) = transcode(UInt16, String(s)) + +_write!(io, ::Nothing) = print(io, "null") +_write!(io, b::Bool) = print(io, b ? "true" : "false") + +function _write!(io, n::Integer) + # I-JSON: only |n| <= 2^53-1 survives a JSON round trip exactly. + -SAFE_INT_MAX <= n <= SAFE_INT_MAX || throw(ArgumentError( + "canonical JSON refuses the integer $n: it is outside the exact " * + "IEEE-754 binary64 range I-JSON allows")) + print(io, n) +end + +function _write!(io, x::AbstractFloat) + _write_float!(io, Float64(x)) +end + +function _write!(io, s::AbstractString) + print(io, '"') + for c in s + _write_char!(io, c) + end + print(io, '"') +end + +# ECMAScript's JSON.stringify escapes exactly these: the two mandatory +# characters, the five with short forms, and every other control +# character as \u00xx with lowercase hex. +function _write_char!(io, c::Char) + if c == '"' + print(io, "\\\"") + elseif c == '\\' + print(io, "\\\\") + elseif c == '\b' + print(io, "\\b") + elseif c == '\t' + print(io, "\\t") + elseif c == '\n' + print(io, "\\n") + elseif c == '\f' + print(io, "\\f") + elseif c == '\r' + print(io, "\\r") + elseif c < ' ' + print(io, "\\u", string(UInt32(c); base=16, pad=4)) + else + print(io, c) + end +end + +# AbstractDict (Dict and JSON3.Object alike): sorted keys, then each pair. +function _write!(io, d::AbstractDict) + print(io, '{') + first = true + for k in sort!(collect(String.(keys(d))); by = _utf16_units) + first || print(io, ',') + first = false + _write!(io, string(k)) + print(io, ':') + _write!(io, d[k]) + end + print(io, '}') +end + +# AbstractVector (Vector and JSON3.Array alike): order preserved. +function _write!(io, a::AbstractVector) + print(io, '[') + for (i, v) in enumerate(a) + i > 1 && print(io, ',') + _write!(io, v) + end + print(io, ']') +end + +# RFC 8785 §3.2.2.3: ECMAScript's Number::toString. Julia's own repr gives +# the shortest round-trip digits (Ryu, the same family ES specifies), so +# the work is reformatting those digits under ES's notation rules. +function _write_float!(io, x::Float64) + isnan(x) && throw(ArgumentError( + "canonical JSON refuses NaN: no consumer reads it back as the same value")) + isinf(x) && throw(ArgumentError( + "canonical JSON refuses Inf: no consumer reads it back as the same value")) + if x == 0 + print(io, "0") # +0.0 and -0.0 both collapse to 0, as ECMAScript does. + return + end + if signbit(x) + print(io, '-') + x = -x + end + digits, n = _shortest_digits(x) + k = length(digits) + if n > 21 || n <= -6 + # Exponential: one digit, then the fraction, then e with an + # explicit sign (ES prints "1e+30", not "1e30"). + print(io, digits[1]) + k > 1 && (print(io, '.'); print(io, digits[2:end])) + e = n - 1 + print(io, 'e', e < 0 ? '-' : '+', string(abs(e))) + elseif n >= k + # An integer too big for the fraction: digits followed by zeros. + print(io, digits, "0"^(n - k)) + elseif n > 0 + # Digits with the point inside them. + print(io, digits[1:n], '.', digits[n+1:end]) + else + # Below one, positional as far as ES goes: 0.000001, not 1e-6. + print(io, "0.", "0"^(-n), digits) + end +end + +# The shortest round-trip decimal form of x as (digits, n) with +# x = 0.digits * 10^n: repr yields either "123", "4.5" or "1.5e-10". +function _shortest_digits(x::Float64) + s = repr(x) + mantissa, exponent = occursin('e', s) ? (split(s, 'e') .|> String) : (s, "") + e = isempty(exponent) ? 0 : parse(Int, exponent) + if occursin('.', mantissa) + intpart, fracpart = split(mantissa, '.') .|> String + else + intpart, fracpart = mantissa, "" + end + digits = string(intpart, fracpart) + n = length(intpart) + e + # Trailing zeros are notation, not value: 1.0e30 is digits "1", n 31. + while length(digits) > 1 && endswith(digits, '0') + digits = digits[1:end-1] + end + digits, n +end + +end # module diff --git a/src/core/memory_probe.jl b/src/core/memory_probe.jl index 6406c37a..a4ad4304 100644 --- a/src/core/memory_probe.jl +++ b/src/core/memory_probe.jl @@ -50,8 +50,16 @@ mutable struct TaskTrace samples :: Int # samples that found at least one live process peak_kib :: Int hwm_kib :: Int + # The sample window the task's children were live in: the index of the first + # sample taken after the first of them started, and the last taken before + # the last was waited for. 0 means never. Two tasks overlap when the later + # one's `first_sample` falls at or before the earlier one's `last_sample`, + # which is the interleaving the grade algebra sequences over. + first_sample :: Int + last_sample :: Int end -TaskTrace(label::AbstractString) = TaskTrace(String(label), Int[], 0, 0, 0, 0) +TaskTrace(label::AbstractString) = + TaskTrace(String(label), Int[], 0, 0, 0, 0, 0, 0) mutable struct Probe interval :: Float64 @@ -269,7 +277,10 @@ function watch!(proc::Base.Process) end lock(p.lock) do t.processes += 1 - pid === nothing || push!(t.live, pid) + t.processes == 1 && (t.first_sample = p.samples + 1) + if pid !== nothing + push!(t.live, pid) + end end pid === nothing || _sample!(p) pid @@ -301,16 +312,235 @@ function unwatch!(pid::Int) p === nothing && return nothing lock(p.lock) do filter!(!=(pid), t.live) + isempty(t.live) && (t.last_sample = p.samples) + end + nothing +end + +## Grading a stage against the high-water-mark algebra +# `docs/occupancy` reads a stage as a timeline of grades and checks the peak the +# algebra predicts against the one the probe measured. The grade is the pinned +# HWM instance's (`occupancy-types`' sibling `resource-algebra`, +# `ResourceAlgebra.Instances.HWM`; both are fetched by `scripts/fetch-siblings.sh`): +# +# hwm peak drift with drift ≤ peak +# +# * `peak` — the high-water mark this interaction reached, in KiB; +# * `drift` — the change in the live peak sum across the interaction's +# boundary, so a boundary that also releases an earlier task is negative; +# * `release` = peak − drift — everything freed between the interaction's start +# and the next boundary, which can exceed the interaction's own peak. +# +# A task's drift subtracts earlier tasks that exited before the next start, so it +# can be negative — the signed-drift extension `Instances/HWM`'s header records +# as H-2. The folded drift is the sum of peaks still live at the stage's end. +# +# A task's drift subtracts earlier tasks that exited before the next start, so +# it can be negative. The folded drift is the sum of peaks still live there. +# +# Sequencing starts the next interaction from the previous one's drift: +# +# hwm p₁ n₁ ⊗ hwm p₂ n₂ = hwm (max p₁ (n₁ + p₂)) (n₁ + n₂) +# +# and `OccupancyTypes.Reclaim` proves of that definition +# +# peak (a ⊗ b) = max (peak a) (drift a + peak b) (reclaim-law) +# peak (a ⊗ b) ≤ peak a + peak b (occupancy≤cost) +# drift a = peak a ⇒ peak (a ⊗ c) = peak a + peak c (no-reclaim) +# drift a < peak a, 0 < peak c ⇒ peak (a ⊗ c) < peak a + peak c (reclaimed) +# +# The last two are the two shapes a stage actually has. Run together, the first +# task is still resident when the second starts (drift = peak) and the stage peak +# is the sum; run one after the other it has exited (drift = 0) and the stage peak +# is the larger. The samples already tell the two apart, so the drift is read from +# the task windows and never assumed. +# +# The prediction is an upper bound on the measurement: at a handover the task +# still live holds at most its peak, so the drift the algebra adds is at least +# what a sample could have seen. `disagreement` therefore reports only what that +# bound forbids — a measured stage peak above the prediction — which is the "one +# of them is a bug" the occupancy reading asks about. + +const GRADE_ALGEBRA = "ResourceAlgebra.Instances.HWM" + +"A grade of the pinned high-water-mark algebra: a peak and a net memory change." +struct Grade + peak :: Int + drift :: Int + function Grade(peak::Integer, drift::Integer) + drift <= peak || throw(ArgumentError( + "a grade holds at most what it reached: drift $drift > peak $peak")) + new(Int(peak), Int(drift)) + end +end + +"`release(grade)` — `peak − drift`: everything freed by the interaction's boundary." +release(g::Grade) = g.peak - g.drift + +"`seq(a, b)` — sequence `b` after `a`: the suffix starts from `a`'s drift." +seq(a::Grade, b::Grade) = Grade(max(a.peak, a.drift + b.peak), a.drift + b.drift) + +"`alt(a, b)` — the larger grade, componentwise (what the timeline never needs, kept for the pair of laws)." +alt(a::Grade, b::Grade) = Grade(max(a.peak, b.peak), max(a.drift, b.drift)) + +"Nothing allocated and nothing held: the algebra's unit." +const NO_GRADE = Grade(0, 0) + +# The trace's sample window, or `nothing` for a record that carries none (one +# written before the windows were sampled). +function _window(rec::AbstractDict) + first_seen = get(rec, "first_sample", nothing) + last_seen = get(rec, "last_sample", nothing) + (first_seen isa Integer && last_seen isa Integer) ? (Int(first_seen), Int(last_seen)) : nothing +end + +_is_windowed(rec::AbstractDict) = _window(rec) !== nothing + +# Whether `prev` was still live when `next` started, so the handover hands its +# peak on rather than reclaiming it. A started `prev` never unwatched (window +# end 0) was still live; a task that never started (window start 0) never overlapped. +_overlaps(prev::Tuple{Int,Int}, next::Tuple{Int,Int}) = + prev[1] != 0 && next[1] != 0 && (prev[2] == 0 || next[1] <= prev[2]) + +# Grades and labels in sampled order, or `nothing` when the record carries no +# windows: a comparison is then skipped rather than made on an assumed order. +function _grades_and_labels(occ::AbstractDict) + recs = get(occ, "tasks", nothing) + recs isa AbstractVector || return Grade[], String[] + all(_is_windowed, recs) || return nothing + # Sampled order, ties broken by record order so the result never depends on + # the sort's stability. + order = sort(collect(eachindex(recs)); by = i -> (_window(recs[i])[1], i)) + ordered = collect(recs)[order] + windows = [_window(r) for r in ordered] + peaks = [Int(get(r, "peak_rss_kib", 0)) for r in ordered] + grades = Grade[] + labels = String[] + live = Int[] + held = 0 + for (i, rec) in enumerate(ordered) + push!(live, i) + if i == length(ordered) + # The stage waits for its children: a task still resident at the end + # keeps its peak in the folded drift, an exited one holds nothing. + filter!(j -> windows[j][1] != 0 && windows[j][2] == 0, live) + else + filter!(j -> _overlaps(windows[j], windows[i + 1]), live) + end + remaining = sum(j -> peaks[j], live; init = 0) + # seq adds drifts, so reclaim expired predecessors as well as this task. + push!(grades, Grade(peaks[i], remaining - held)) + held = remaining + push!(labels, String(get(rec, "label", "?"))) end + grades, labels +end + +""" + task_grades(occupancy) -> Union{Vector{Grade},Nothing} + +One grade per task of a stage's `occupancy` record, in sampled order. Its drift +is the change in the live peak sum by the next task's start, including releases +of earlier tasks; the final boundary retains only unclosed windows. Folding +with `seq` carries exactly the peaks still live at each boundary. `nothing` when +the record carries no sample windows. +""" +function task_grades(occ::AbstractDict) + graded = _grades_and_labels(occ) + graded === nothing ? nothing : graded[1] +end + +""" + stage_grade(occupancy) -> Union{Grade,Nothing} + +The stage's grade: its tasks' grades folded with `seq`, the algebra's sequencing. +`nothing` when the record carries no sample windows. +""" +function stage_grade(occ::AbstractDict) + grades = task_grades(occ) + grades === nothing && return nothing + isempty(grades) ? NO_GRADE : foldl(seq, grades) +end + +"`predicted_peak_rss_kib(occupancy)` — the peak the algebra predicts, or `nothing`." +function predicted_peak_rss_kib(occ::AbstractDict) + g = stage_grade(occ) + g === nothing ? nothing : g.peak +end + +"`cost_bound_rss_kib(occupancy)` — the peaks added: the peak if no cut reclaimed anything (occupancy≤cost's right-hand side)." +function cost_bound_rss_kib(occ::AbstractDict) + grades = task_grades(occ) + grades === nothing ? nothing : sum(g -> g.peak, grades; init = 0) +end + +"`reclaimed_kib(occupancy)` — the slack between the cost bound and the predicted peak: what the cuts reclaim." +function reclaimed_kib(occ::AbstractDict) + cost = cost_bound_rss_kib(occ) + predicted = predicted_peak_rss_kib(occ) + (cost === nothing || predicted === nothing) ? nothing : cost - predicted +end + +""" + disagreement(occupancy) -> Union{String,Nothing} + +Why the measured stage peak contradicts the grade algebra, or `nothing` when it +does not. The prediction is an upper bound, so the one contradiction is a +measured peak above it: a sample then saw more resident memory than the timed +timeline of tasks can hold, and one of the two is wrong. A record with no +windows, or none the probe graded, gives `nothing`: not measured is not a +disagreement. +""" +function disagreement(occ::AbstractDict) + predicted = predicted_peak_rss_kib(occ) + measured = get(occ, "child_peak_rss_kib", nothing) + (predicted === nothing || !(measured isa Integer)) && return nothing + measured > predicted && return "measured $(measured) KiB above the predicted $(predicted) KiB" nothing end +""" + graded(occupancy) -> Union{OrderedDict,Nothing} + +The grades of a stage's `occupancy` record, ready to sit beside it in the +attestation: per task `peak_kib`/`drift_kib`/`release_kib`, the stage's +`predicted_peak_kib`, the `cost_bound_kib` (its peaks added) and the +`reclaimed_kib` slack between the two, the `measured_peak_kib` it is checked +against, and the `disagreement` between them or `nothing`. `nothing` when the +record carries no sample windows. +""" +function graded(occ::AbstractDict) + graded_pair = _grades_and_labels(occ) + graded_pair === nothing && return nothing + grades, labels = graded_pair + stage = isempty(grades) ? NO_GRADE : foldl(seq, grades) + cost = sum(g -> g.peak, grades; init = 0) + OrderedDict{String,Any}( + "algebra" => GRADE_ALGEBRA, + "tasks" => Any[OrderedDict{String,Any}( + "label" => labels[i], + "peak_kib" => grades[i].peak, + "drift_kib" => grades[i].drift, + "release_kib" => release(grades[i])) + for i in eachindex(grades)], + "predicted_peak_kib" => stage.peak, + "drift_kib" => stage.drift, + "release_kib" => release(stage), + "cost_bound_kib" => cost, + "reclaimed_kib" => cost - stage.peak, + "measured_peak_kib" => get(occ, "child_peak_rss_kib", nothing), + "disagreement" => disagreement(occ), + ) +end + _task_record(t::TaskTrace) = OrderedDict{String,Any}( "label" => t.label, "processes" => t.processes, "samples" => t.samples, "peak_rss_kib" => t.peak_kib, "hwm_kib" => t.hwm_kib, + "first_sample" => t.first_sample, + "last_sample" => t.last_sample, ) """ @@ -337,6 +567,11 @@ function finish!(p::Probe) "tasks" => Any[_task_record(t) for t in traces], ) p.error === nothing || (p.record["error"] = p.error) + # The grades the sample windows imply, beside the samples they came from. A + # record whose tasks carry no windows cannot be graded, and is left as it is + # rather than graded on an assumed order. + g = graded(p.record) + g === nothing || (p.record["grades"] = g) p.record end diff --git a/src/pipeline/dada2/dada2_functions.r b/src/pipeline/dada2/dada2_functions.r index 8792317a..5d61fa6c 100644 --- a/src/pipeline/dada2/dada2_functions.r +++ b/src/pipeline/dada2/dada2_functions.r @@ -15,8 +15,6 @@ # https://creativecommons.org/licenses/by/4.0/. library(dada2) -library(dplyr) -library(tibble) # library(yaml) # BiocParallel defaults to MulticoreParam, which forks. ShortRead::qa(), reached @@ -214,10 +212,28 @@ write_fasta <- function(seq_table, tables_dir, prefix) { # Writes taxonomy assignments to CSV. Bootstrap confidence values (0-100 per # taxonomic rank) # Returns taxa_df (SeqName + Sequence + taxonomy), used by write_combined_table(). +# +# dplyr::left_join(x, y, by) keeps x's rows in x's order with x's columns first +# and then y's (the key once). merge() alone sorts the rows and reorders the +# columns, so both are restored explicitly: a temporary position column restores +# x's row order, and the column order is rebuilt from x's columns plus y's non-key +# ones. The CI oracle checks the written CSVs byte-for-byte against the previous output. +base_left_join <- function(x, y, by) { + x_cols <- colnames(x) + position <- ".left_join_position" + while (position %in% c(colnames(x), colnames(y))) position <- paste0(position, "_") + x[[position]] <- seq_len(nrow(x)) + joined <- merge(x, y, by = by, all.x = TRUE, sort = FALSE) + joined <- joined[order(joined[[position]]), + c(x_cols, setdiff(colnames(y), by)), drop = FALSE] + rownames(joined) <- NULL + joined +} + write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, prefix) { taxa_df <- as.data.frame(tax_matrix, stringsAsFactors = FALSE) taxa_df$Sequence <- rownames(taxa_df) - taxa_df <- dplyr::left_join(index, taxa_df, by = "Sequence") + taxa_df <- base_left_join(index, taxa_df, by = "Sequence") write.csv(taxa_df, file.path(tables_dir, paste0(prefix, ".csv")), quote = FALSE, row.names = FALSE) @@ -225,11 +241,11 @@ write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, prefix) boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) colnames(boot_df) <- paste0(colnames(boot_df), "_boot") boot_df$Sequence <- rownames(boot_matrix) - boot_df <- dplyr::left_join(index, boot_df, by = "Sequence") + boot_df <- base_left_join(index, boot_df, by = "Sequence") write.csv(boot_df, file.path(tables_dir, paste0(prefix, "_bootstraps.csv")), quote = FALSE, row.names = FALSE) - combined_df <- dplyr::left_join(taxa_df, boot_df, by = c("SeqName", "Sequence")) + combined_df <- base_left_join(taxa_df, boot_df, by = c("SeqName", "Sequence")) write.csv(combined_df, file.path(tables_dir, paste0(prefix, "_combined.csv")), quote = FALSE, row.names = FALSE) @@ -239,10 +255,13 @@ write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, prefix) # Joins taxonomy with per-sample counts write_combined_table <- function(taxa_df, index, seq_table, tables_dir, tax_filename, asv_filename) { seq_t <- as.data.frame(t(seq_table), stringsAsFactors = FALSE) - seq_t <- tibble::rownames_to_column(seq_t, "Sequence") - write.csv(dplyr::left_join(taxa_df, seq_t, by = "Sequence"), + # tibble::rownames_to_column(seq_t, "Sequence") put the row names in a leading + # column and dropped them from the row names; data.frame() does the same. + seq_t <- data.frame(Sequence = rownames(seq_t), seq_t, row.names = NULL, + check.names = FALSE, stringsAsFactors = FALSE) + write.csv(base_left_join(taxa_df, seq_t, by = "Sequence"), file.path(tables_dir, tax_filename), quote = FALSE, row.names = FALSE) - write.csv(dplyr::left_join(index, seq_t, by = "Sequence"), + write.csv(base_left_join(index, seq_t, by = "Sequence"), file.path(tables_dir, asv_filename), quote = FALSE, row.names = FALSE) invisible(NULL) } diff --git a/src/server/routes/analysis.jl b/src/server/routes/analysis.jl index 0dd506a3..6154f5b7 100644 --- a/src/server/routes/analysis.jl +++ b/src/server/routes/analysis.jl @@ -1203,7 +1203,8 @@ function _differential_config(study::String) DifferentialConfig(String(val("analysis.differential.offset", "tss")), Float64(val("analysis.differential.min_prevalence", 0.0)); method=String(val("analysis.differential.method", "nb_glm")), - replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65))) + replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65)), + dispersion=String(val("analysis.differential.dispersion", "ml"))) end """ @@ -1212,7 +1213,11 @@ end Run a differential abundance test between exactly two conditions (runs or pooled-run sub-groups) and return the per-taxon table with a volcano plot. The first condition is the reference, so a positive estimate means more abundant in -the second. Every refusal is an explicit JSON error. +the second: a natural-log fold change for `nb_glm`, or a mean CLR difference for +`clr_welch`. `ArgumentError`s from config construction or the test return 400; +scaling refusals and analysis `ErrorException`s return 422, and missing MASS or +zCompositions returns 503. Missing studies or results return 404. +`RBusyError` and other uncaught errors propagate to the caller. """ function _differential(study::String, body) study in _study_names() || return json_error(404, "study_not_found", diff --git a/src/server/routes/config.jl b/src/server/routes/config.jl index 2f927ba1..20b55a60 100644 --- a/src/server/routes/config.jl +++ b/src/server/routes/config.jl @@ -125,8 +125,10 @@ const _NULLABLE_NUMBERS = ("phylogeny.placement.place.heuristic",) """ _config_value_error(key, value) -> Union{String,Nothing} -Why a value cannot be saved under this key, or nothing. The type follows the -factory default; a key whose default is ~ accepts a string, number or boolean. +Return a rejection message for `value` under the dotted pipeline `key`, or +`nothing`. Enums, command arguments, remote settings and file-name parts have +specific checks; otherwise the type follows the factory default, with no type +restriction for a default of ~. Does not write config or check key recognition. """ function _config_value_error(key::String, value) v = value isa JSON3.Array ? collect(value) : value @@ -292,6 +294,13 @@ end # Read the document; an unparseable file becomes a 400 naming it. PrimersLibrary.load # raises on a bad file because an empty document would pass validation and the next # Save would overwrite the real primers. Returns the document, or an HTTP.Response error. +""" + _read_primers() -> Union{Dict,HTTP.Response} + +Read the canonical primers document, or an empty document if the file is absent. +Loading errors become a 400 `primers_unreadable` response naming the file and +reason; the file is not modified. +""" function _read_primers() try PrimersLibrary.load(_primers_path()) diff --git a/src/server/routes/databases.jl b/src/server/routes/databases.jl index 2d22fd0e..6ff74393 100644 --- a/src/server/routes/databases.jl +++ b/src/server/routes/databases.jl @@ -15,6 +15,13 @@ end # Read the document; an unparseable file becomes a 400 naming it. DatabasesLibrary.load # raises on a bad file because an empty document would pass validation and the next # Save would overwrite the real config. Returns the document, or an HTTP.Response error. +""" + _read_databases() -> Union{Dict,HTTP.Response} + +Read the canonical databases document, or an empty document if the file is absent. +Loading errors become a 400 `databases_unreadable` response naming the file and +reason; the file is not modified. +""" function _read_databases() try DatabasesLibrary.load(_databases_path()) diff --git a/src/server/server.jl b/src/server/server.jl index cef70d33..880a3f84 100644 --- a/src/server/server.jl +++ b/src/server/server.jl @@ -221,11 +221,15 @@ module Server _allowed_origins() -> Set{String} The browser origins allowed to call the API cross-origin: the origin this - server is served from (its bound port, spelled with `localhost` and + server is served from over HTTP (its bound port, spelled with `localhost` and `127.0.0.1`, plus the bound host itself when it is a specific name), the Vite dev server, and any listed comma-separated in `JULIA_METAMANIFOLD_ALLOWED_ORIGINS`. Every other origin is refused, including other ports and schemes on localhost. + + Wildcard bind addresses are not included. Read the environment on each call, + ignoring blank entries and normalising case, surrounding whitespace and + trailing slashes; entries are not validated as URLs. """ function _allowed_origins() port = _bound_port[] @@ -241,12 +245,43 @@ module Server allowed end + _is_wildcard(h::AbstractString) = lowercase(h) in ("0.0.0.0", "::", "[::]") + + """ + _bound_ip_origin(origin, host_header) -> Bool + + Whether `origin` names the exact IP address the browser reached a wildcard + bind on: an `http://` origin whose host is an IP literal (not a name), whose + port is the bound port, and whose host-and-port equals the request's `Host` + header. A browser only sends such an `Origin` for a page it loaded from that + address, so the request is same-origin by construction. A DNS-rebinding page + arrives with the attacker's domain as both `Host` and `Origin`; a name is not + an IP literal, so it is refused here as before. Hostnames (e.g. + `http://lab-box:8080`) and any address behind a proxy on another port must be + listed in `JULIA_METAMANIFOLD_ALLOWED_ORIGINS`. + """ + function _bound_ip_origin(origin::AbstractString, host_header::AbstractString) + o = _normalise_origin(origin) + startswith(o, "http://") || return false + hostport = o[length("http://") + 1:end] + hostport == lowercase(strip(host_header)) || return false + m = match(r"^(?:\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-f:]+\]):(\d+)$"i, hostport) + !isnothing(m) && parse(Int, m.captures[1]) == _bound_port[] + end + """ _cors_middleware(next) - Refuse requests whose Host is not loopback on a loopback bind, and requests - carrying an `Origin` outside `_allowed_origins()`. Allowed origins get CORS - headers and preflight answers; requests with no `Origin` pass unchanged. + Return a request handler that answers 403 for a non-empty, non-loopback Host + on a loopback bind, or a non-empty `Origin` outside `_allowed_origins()` after + normalisation. On a wildcard bind an origin naming the request's own IP host + on the bound port is also admitted (`_bound_ip_origin`): the UI is then + served, and writable, at whatever address the machine is reached on, without + trusting a hostname from the `Host` header. The Host check also applies to + requests without an Origin. Allowed origins get CORS headers echoing the + supplied Origin; OPTIONS gets 204 without calling `next`. Otherwise call + `next(req)`, adding CORS headers only when the Origin is non-empty. Errors + from `next` propagate. """ function _cors_middleware(next) function(req::HTTP.Request) @@ -261,6 +296,8 @@ module Server return next(req) end _normalise_origin(origin) in _allowed_origins() || + (_is_wildcard(_bound_host[]) && + _bound_ip_origin(origin, HTTP.header(req, "Host", ""))) || return HTTP.Response(403, "Forbidden: origin not allowed") cors_headers = [ "Access-Control-Allow-Origin" => origin, @@ -292,8 +329,13 @@ module Server _file_middleware(next) Serve run outputs under `/files/{study}/runs/...` from the projects tree and - the frontend build for every other non-API path, passing `/api/` requests on. + the frontend build for other non-API paths, passing `/api/` requests to `next`. + Return a request handler; query strings are ignored when locating files. + Missing run files return 404; existing files resolving outside the projects + tree return 403. For other paths, serve a matching frontend file or fall back + to `index.html`, calling `next` if neither exists. Run files are sent with `nosniff`; report uploads also get `_UPLOAD_CSP`. + File resolution and read errors, and errors from `next`, propagate. """ function _file_middleware(next) function(req::HTTP.Request) @@ -396,6 +438,19 @@ module Server # above, so they would bypass the Host and Origin checks; nothing uses them. const _SERVE_OPTIONS = (access_log=nothing, show_errors=false, docs=false, metrics=false) + """ + start(; root=pwd(), host="127.0.0.1", port=8080, listen=true) + + Set the server's root for `data/`, `projects/` and `config/`, record the bind + host and port for request guards, and initialise missing study projects. + Individual project creation failures are caught and startup continues; + errors enumerating studies propagate. + + With `listen=false`, perform this initialisation and return `nothing` without + opening a listener. Otherwise serve the API and frontend with the request + guards enabled and Oxygen docs and metrics disabled, waiting until the server + stops and returning its server object. Listener startup errors propagate. + """ function start(; root=pwd(), host="127.0.0.1", port=8080, listen::Bool=true) global_logger(_SuppressEpipe(global_logger())) ServerState.set_root!(root) diff --git a/test/fixtures/jcs_oracle.jsonl b/test/fixtures/jcs_oracle.jsonl new file mode 100644 index 00000000..a61e6e37 --- /dev/null +++ b/test/fixtures/jcs_oracle.jsonl @@ -0,0 +1,11 @@ +{"input":{"b":1,"a":2,"c":[]},"expected":"{\"a\":2,\"b\":1,\"c\":[]}"} +{"input":{"€":"euro","Ａ":"fullwidth A","a":"first"},"expected":"{\"a\":\"first\",\"€\":\"euro\",\"Ａ\":\"fullwidth A\"}"} +{"input":{"😀":"astral first?","Ａ":"high bmp"},"expected":"{\"😀\":\"astral first?\",\"Ａ\":\"high bmp\"}"} +{"input":{"d":[3,1,2],"e":{"z":1,"y":{"x":[true,false,null]}}},"expected":"{\"d\":[3,1,2],\"e\":{\"y\":{\"x\":[true,false,null]},\"z\":1}}"} +{"input":[4.50,24499999999999998,1E+30,0.000001,1E-7,1E+21,1E+20,1E-33,-0.0,0.0,9.9999999999999997E+22],"expected":"[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,1e-33,0,0,1e+23]"} +{"input":[0.000001,1E-7,1.5E-10,9007199254740991,-42],"expected":"[0.000001,1e-7,1.5e-10,9007199254740991,-42]"} +{"input":{"s\":with\"quotes":"tab\there\nnewline","esc\\":"\\backslash","ctl":"\u0001\u001f"},"expected":"{\"ctl\":\"\\u0001\\u001f\",\"esc\\\\\":\"\\\\backslash\",\"s\\\":with\\\"quotes\":\"tab\\there\\nnewline\"}"} +{"input":{"nested":{"arr":[{"n":1.0},{"m":2}],"deep":{"deeper":{"deepest":"value"}}}},"expected":"{\"nested\":{\"arr\":[{\"n\":1},{\"m\":2}],\"deep\":{\"deeper\":{\"deepest\":\"value\"}}}}"} +{"input":{"literals":true,"n":null,"f":false,"empty_obj":{},"empty_arr":[]},"expected":"{\"empty_arr\":[],\"empty_obj\":{},\"f\":false,\"literals\":true,\"n\":null}"} +{"input":[5E-324,1.7976931348623157E+308,2.2250738585072014E-308,0.3,0.1,0.30000000000000004],"expected":"[5e-324,1.7976931348623157e+308,2.2250738585072014e-308,0.3,0.1,0.30000000000000004]"} +{"input":{"Plotly":{"layout":{"title":{"text":"CO₂"}},"data":[{"x":[1,2,3],"y":[4.5,5.5,6.5],"type":"scatter"}]}},"expected":"{\"Plotly\":{\"data\":[{\"type\":\"scatter\",\"x\":[1,2,3],\"y\":[4.5,5.5,6.5]}],\"layout\":{\"title\":{\"text\":\"CO₂\"}}}}"} diff --git a/test/integration/test_pipeline.jl b/test/integration/test_pipeline.jl index bd3b5d6f..f3a42bc0 100644 --- a/test/integration/test_pipeline.jl +++ b/test/integration/test_pipeline.jl @@ -261,7 +261,7 @@ run_dir(run) = joinpath(PROJECT_ROOT, "projects", PROJECT_NAME, run) probe_cfg = Dict("occupancy_probe" => Dict("enabled" => true, "interval_ms" => 10)) old_root = SV.ServerState._root[] - written = String[] + original_configs = Dict{String,Union{Nothing,Vector{UInt8}}}() try SV.ServerState.set_root!(PROJECT_ROOT) @@ -269,9 +269,18 @@ rm(joinpath(run_dir("run_A"), "vsearch"); recursive=true, force=true) rm(joinpath(run_dir("run_A"), "swarm", "vsearch"); recursive=true, force=true) yaml_a = joinpath(PROJECT_ROOT, "data", PROJECT_NAME, "run_A", "pipeline.yml") + original_configs[yaml_a] = isfile(yaml_a) ? read(yaml_a) : nothing YAML.write_file(yaml_a, probe_cfg) - push!(written, yaml_a) - SV._run_stage(PROJECT_NAME, "run_A", "vsearch", db_config, dbs_loaded) + response = SV.Oxygen.internalrequest(SV.HTTP.Request( + "POST", "/api/v1/studies/$PROJECT_NAME/runs/run_A/stages/vsearch")) + @test response.status == 200 + job = SV.JobQueue.get_job(String(JSON3.read(String(response.body)).id)) + @test !isnothing(job) + task = lock(SV.JobQueue._jobs_lock) do + job.task + end + isnothing(task) || wait(task) + @test job.status == SV.JobQueue.complete @test isfile(joinpath(run_dir("run_A"), "vsearch", "taxonomy.tsv")) @test isfile(joinpath(run_dir("run_A"), "swarm", "vsearch", "taxonomy.tsv")) @@ -279,8 +288,8 @@ rm(joinpath(run_dir("run_B"), "vsearch"); recursive=true, force=true) rm(joinpath(run_dir("run_B"), "swarm", "vsearch"); recursive=true, force=true) yaml_b = joinpath(PROJECT_ROOT, "data", PROJECT_NAME, "run_B", "pipeline.yml") + original_configs[yaml_b] = isfile(yaml_b) ? read(yaml_b) : nothing YAML.write_file(yaml_b, probe_cfg) - push!(written, yaml_b) merged = SV._run_full_pipeline(PROJECT_NAME, "run_B", db_config, dbs_loaded) @test haskey(merged.tables, "merged") @test isfile(joinpath(run_dir("run_B"), "vsearch", "taxonomy.tsv")) @@ -300,7 +309,9 @@ @test labels == ["vsearch:asv", "vsearch:otu"] end finally - foreach(p -> rm(p; force=true), written) + for (path, contents) in original_configs + isnothing(contents) ? rm(path; force=true) : write(path, contents) + end SV.ServerState._root[] = old_root end end diff --git a/test/occupancy_disagreement_test.jl b/test/occupancy_disagreement_test.jl index 00837263..15df70fd 100644 --- a/test/occupancy_disagreement_test.jl +++ b/test/occupancy_disagreement_test.jl @@ -341,6 +341,25 @@ _task(rec, label) = only(t for t in rec["tasks"] if t["label"] == label) @info "occupancy: concurrent stage peak vs the larger serial stage peak" concurrent = conc["child_peak_rss_kib"] serial = serial_max overlapped = overlap cost_spread = round(spread; sigdigits = 3) overlap || @warn "occupancy: the concurrent searches did not overlap in memory on this runner, so this run cannot show the order dependence" + # The grades the sample windows imply, on this same run + # (fork issue #105). The peak is recorded, not asserted — a + # runner that never has both searches resident cannot show it — + # but the algebra's own claims are asserted of every order: + # every task is timed, the prediction at least reaches what the + # samples saw (the prediction is an upper bound: at a handover + # the live task holds at most its peak), it never exceeds the + # cost bound (Reclaim.occupancy≤cost), and the slack between + # them is never negative. + for o in ORDERS + r = records[o] + @test all(t -> haskey(t, "first_sample") && haskey(t, "last_sample"), r["tasks"]) + @test haskey(r, "grades") + @test MP.predicted_peak_rss_kib(r) >= r["child_peak_rss_kib"] + @test MP.predicted_peak_rss_kib(r) <= MP.cost_bound_rss_kib(r) + @test MP.reclaimed_kib(r) >= 0 + @test MP.disagreement(r) === nothing + end + path = get(ENV, "OCCUPANCY_RESULTS", "") if !isempty(path) io = IOBuffer() diff --git a/test/runtests.jl b/test/runtests.jl index ed351f13..77e15bcf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -45,6 +45,7 @@ const UNIT_FILES = [ "test_remote_stages.jl", "test_jobs.jl", "test_provenance.jl", + "test_occupancy_grade.jl", "test_install_pins.jl", "test_migrate_composition.jl", "test_sample_reads.jl", @@ -53,6 +54,7 @@ const UNIT_FILES = [ "test_permanova.jl", "test_write_gates.jl", "test_server_security.jl", + "test_canonical.jl", ] # The files named on the command line ("test_routes.jl", "routes" or diff --git a/test/unit/test_canonical.jl b/test/unit/test_canonical.jl new file mode 100644 index 00000000..1a9bb01b --- /dev/null +++ b/test/unit/test_canonical.jl @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Unit tests for the JCS (RFC 8785) canonical JSON writer. The oracle is the +# estate's ijson-jcs crate (Rust): every fixture in +# test/fixtures/jcs_oracle.jsonl was PRODUCED by that crate, and this file +# asserts MetaManifold.Canonical reproduces it byte-for-byte. Any +# disagreement is a bug in one of the two implementations, and the fixture +# names the case. +using MetaManifold +using MetaManifold.Canonical: jcs_bytes, jcs_sha256 +using JSON3, SHA, Test + +@testset "canonical JSON (JCS)" begin + # Byte-for-byte against the Rust crate's output, over the fixture + # corpus: key sorting (including the astral-before-high-BMP UTF-16 + # case), ES number formatting, string escaping, nesting, literals. + fixtures = joinpath(@__DIR__, "..", "fixtures", "jcs_oracle.jsonl") + for (i, line) in enumerate(eachline(fixtures)) + case = JSON3.read(line) + @testset "oracle case $i" begin + value = case.input + # The input arrives already parsed; the expected is the oracle's text. + @test String(jcs_bytes(value)) == case.expected + end + end + + @testset "key order is irrelevant, bytes are not" begin + a = Dict("b" => 1, "a" => Dict("y" => [1.0, 2], "x" => "s")) + b = Dict("a" => Dict("x" => "s", "y" => [1.0, 2]), "b" => 1) + @test jcs_bytes(a) == jcs_bytes(b) + # ...but array order is preserved, as JCS requires. + c = Dict("a" => [2, 1]) + @test jcs_bytes(a) != jcs_bytes(c) + end + + @testset "the astral key sorts before the high BMP key" begin + # U+1F600 arrives as the surrogate pair D83D DE00; UTF-16 code-unit + # order puts it before U+FF21. Naive code-point order would agree + # here; naive byte order would too -- this case pins the rule. + d = Dict("\uff21" => "high bmp", "\U0001f600" => "astral") + @test String(jcs_bytes(d)) == "{\"😀\":\"astral\",\"Ａ\":\"high bmp\"}" + end + + @testset "float notation follows ECMAScript" begin + @test String(jcs_bytes([4.50, 2.4499999999999998e16, 1e30, 1e-6, 1e-7, + 1e21, 1e20, -0.0, 5e-324])) == + "[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,0,5e-324]" + end + + @testset "refusals" begin + @test_throws ArgumentError jcs_bytes(2^53) + @test_throws ArgumentError jcs_bytes(-(2^53)) + @test_throws ArgumentError jcs_bytes(Inf) + @test_throws ArgumentError jcs_bytes(NaN) + @test jcs_bytes(2^53 - 1) == b"9007199254740991" + end + + @testset "a content address is stable" begin + # The use this exists for: a chart-like spec built in any key + # order hashes identically, and the hash can be recorded where the + # bytes cannot. + spec = Dict( + "layout" => Dict("title" => Dict("text" => "CO₂")), + "data" => [Dict("x" => [1, 2, 3], "y" => [4.5, 5.5, 6.5], + "type" => "scatter")]) + same = Dict( + "data" => [Dict("type" => "scatter", "y" => [4.5, 5.5, 6.5], + "x" => [1, 2, 3])], + "layout" => Dict("title" => Dict("text" => "CO₂"))) + @test jcs_sha256(spec) == jcs_sha256(same) + @test jcs_sha256(spec) == bytes2hex(sha256(jcs_bytes(spec))) + end +end diff --git a/test/unit/test_differential.jl b/test/unit/test_differential.jl index 3b034896..36e98158 100644 --- a/test/unit/test_differential.jl +++ b/test/unit/test_differential.jl @@ -172,6 +172,14 @@ end @test_throws ArgumentError DifferentialConfig(; method="ancombc") # The pre-review name, never released, is not accepted. @test_throws ArgumentError DifferentialConfig(; method="clr_lm") + # The dispersion option: ml is the default, glmGamPoi is the alternative, + # and it is a negative-binomial setting only. + @test DifferentialConfig().dispersion == "ml" + @test DifferentialConfig(dispersion="glmGamPoi").dispersion == "glmGamPoi" + @test DifferentialConfig(dispersion=" GLMGAMPOI ").dispersion == "glmGamPoi" + @test_throws ArgumentError DifferentialConfig(dispersion="shrinkage") + @test_throws ArgumentError DifferentialConfig(method="clr_welch", dispersion="glmGamPoi") + @test DifferentialConfig(method="clr_welch").dispersion == "ml" for d in (0, 1, -0.5, NaN) @test_throws ArgumentError DifferentialConfig(; method="clr_welch", replacement_delta=d) end @@ -720,4 +728,41 @@ end @test rcopy(Bool, reval("requireNamespace('zCompositions', quietly = TRUE)")) == HAVE_ZCOMP end end + + ## Without glmGamPoi, dispersion: glmGamPoi refuses with + # GlmGamPoiUnavailable. Same trick as the zCompositions testset above: a + # `requireNamespace` bound in R's global environment reports glmGamPoi + # missing and defers to base for every other package, so the production + # probe sees the package as absent on any machine. MASS is probed first + # and must be real, so this testset needs it. + @testset "nb_glm without glmGamPoi refuses with GlmGamPoiUnavailable" begin + if !HAVE_MASS + @test_skip false # no MASS means the ml path's own guard fires first + else + reval("""requireNamespace <- function(package, ...) { + if (identical(package, "glmGamPoi")) FALSE + else base::requireNamespace(package, ...) + }""") + try + @test !rcopy(Bool, reval("requireNamespace('glmGamPoi', quietly = TRUE)")) + e = try + fit_nb([5 6; 7 3; 9 4; 4 8], ["A", "A", "B", "B"], [0.0, 0.0, 0.0, 0.0]; + levels=("A", "B"), dispersion="glmGamPoi") + nothing + catch err + err + end + @test e isa GlmGamPoiUnavailable + @test occursin("renv::restore()", sprint(showerror, e)) + # The override defers to base for everything else, so the default + # dispersion must still fit normally with it in place. + r = fit_nb([5 6; 7 3; 9 4; 4 8], ["A", "A", "B", "B"], [0.0, 0.0, 0.0, 0.0]; + levels=("A", "B")) + @test all(r.status .== "ok") + finally + reval("rm(requireNamespace)") + end + @test !rcopy(Bool, reval("exists('requireNamespace', envir = globalenv(), inherits = FALSE)")) + end + end end diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index a3053b97..bfa9bbb5 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -14,8 +14,9 @@ using SHA const REPO_ROOT = normpath(joinpath(@__DIR__, "..", "..")) const PINS_PATH = joinpath(REPO_ROOT, "config", "defaults", "tool_versions.yml") -const INSTALL_JL = joinpath(REPO_ROOT, "install.jl") +const INSTALL_JL = joinpath(REPO_ROOT, "scripts", "install.jl") const CI_PATH = joinpath(REPO_ROOT, ".github", "workflows", "ci.yml") +const WORKFLOWS = joinpath(REPO_ROOT, ".github", "workflows") const MANIFEST = joinpath(REPO_ROOT, "Manifest.toml") const RENV_LOCK = joinpath(REPO_ROOT, "renv.lock") @@ -29,13 +30,11 @@ const EXPECTED_PLATFORMS = ["linux-x86_64", "linux-aarch64", "macos-x86_64", "ma is_sha256(s) = s isa AbstractString && occursin(r"^[0-9a-f]{64}$", s) -"""Return the first step of `job` whose `uses:` names `action`, or nothing.""" -function ci_step_using(job, action) - for step in get(job, "steps", []) - startswith(get(step, "uses", ""), action) && return step - end - return nothing -end +# Allow GitHub's own actions, this fork's actions, and Claude Code, each pinned to +# a full commit SHA. +allowed_action(uses) = + occursin(r"^(actions|hyperpolymath)/[A-Za-z0-9_.-]+(/[A-Za-z0-9_./-]+)?@[0-9a-f]{40}$", uses) || + occursin(r"^anthropics/claude-code-action@[0-9a-f]{40}$", uses) @testset "install pins" begin @@ -187,18 +186,18 @@ end ci = YAML.load_file(CI_PATH) job = ci["jobs"]["test"] - # Julia is the one version CI cannot read from the pin file, because nothing can - # be read before Julia exists. It is therefore duplicated, and this is the test - # that makes the duplication safe. + # Julia is the one runtime CI cannot read from the pin file, because nothing can + # be read before Julia exists. Its archive URL and checksum are therefore + # duplicated in the job's env, and this is the test that makes the duplication + # safe. The pinned URL names the version, which must equal Manifest.toml's + # (tested above). # - # Read from the `Set up Julia` step rather than from a matrix: the matrix was - # removed because GitHub appends a matrix combination to the posted check name - # (see "a required check name is a stable identifier" below). The step is located - # by its `uses:` rather than by index, so reordering the steps cannot make this - # assertion quietly vanish. - setup = ci_step_using(job, "julia-actions/setup-julia") - @test setup !== nothing - @test setup["with"]["version"] == pins["runtimes"]["julia"]["version"] + # Read from the job's env rather than from a matrix: the matrix was removed + # because GitHub appends a matrix combination to the posted check name (see "a + # required check name is a stable identifier" below). + archive = pins["runtimes"]["julia"]["archives"]["linux-x86_64"] + @test job["env"]["JULIA_URL"] == archive["url"] + @test job["env"]["JULIA_SHA256"] == archive["sha256"] # A floating runner would carry the R apt pin, which names a 24.04 build, off to # whatever the next LTS ships. @@ -248,9 +247,32 @@ end @test !occursin("set +e", fetch_code) @test occursin("source scripts/ci/fetch_pinned.sh", runs) - for ref in (raw"$VSEARCH_URL", raw"$SWARM_URL", raw"$FASTQC_URL") + for ref in (raw"$VSEARCH_URL", raw"$SWARM_URL", raw"$FASTQC_URL", + raw"$JULIA_URL", raw"$BUN_URL") @test occursin("fetch_pinned \"$ref\"", runs) end + + # Julia and Bun, which actions used to install, are refused unless they hash + # to their pins. + for (sha, file) in ((raw"$JULIA_SHA256", "julia.tar.gz"), (raw"$BUN_SHA256", "bun.zip")) + @test occursin("echo \"$sha $file\" | sha256sum -c -", runs) + end + end + + @testset "every action is one this account allows" begin + # Anything else ends the whole workflow in startup_failure before any job runs, + # so the run names no failing step. That is how ci.yml failed on every run on + # this fork, from at least 2026-10-02 until it stopped using julia-actions, + # oven-sh/setup-bun and codecov/codecov-action. + files = filter(f -> endswith(f, ".yml") || endswith(f, ".yaml"), readdir(WORKFLOWS)) + @test "ci.yml" in files + @testset "$file" for file in files + for (_, job) in YAML.load_file(joinpath(WORKFLOWS, file))["jobs"], + step in get(job, "steps", []) + uses = get(step, "uses", nothing) + uses === nothing || @test allowed_action(uses) + end + end end @testset "a required check name is a stable identifier" begin @@ -287,3 +309,96 @@ end end end end + +@testset "the root keeps its shape" begin + # Adapted from the RSR root-cleanup doctrine (rsr-template-repo, + # docs/proposals/root-cleanup.adoc): the repository root holds authority + # files, the entry points a newcomer runs, tool-mandated build metadata, and + # one directory per concern -- nothing else. The check is BIDIRECTIONAL, + # because a one-directional allowlist only ever ratchets open: a tracked + # root entry that is not listed here is drift, and a listed entry that has + # gone missing is drift too. The tracked tree is what is checked, so local + # untracked build output (bin/, web/) does not fail a developer's run. To + # add a root entry, add it here with its reason in the same commit. + root_entries = [ + # Authority files. GitHub reads CITATION.cff from the default-branch + # root only, so it cannot move. + "README.md", "LICENSE", "CITATION.cff", + # The two entry points a newcomer runs (the launcher stays at the + # root); the installer itself lives under scripts/. + "install.sh", "start.sh", "launcher.sh", + # The RSR scheme's task entry and changelog index. + "Justfile", "CHANGELOG.md", + # Build metadata rooted by its tooling: Julia Pkg, and renv, which + # also scans R/ for dependency discovery. + "Project.toml", "Manifest.toml", "renv.lock", "renv", ".Rprofile", "R", + # One directory per concern. Health files (CONTRIBUTING.md) and CI + # configs (codecov.yml) live under .github/. + ".github", "bench", "config", "data", "docs", "frontend", + "proofs", "scripts", "src", "test", + # Agent instructions (the agents.md convention): coding agents load + # this at startup, and it only works from the root, which is why it + # is listed here rather than living under .github/ with the other + # health files. + "AGENTS.md", + # VCS housekeeping, and the editor configuration. + ".gitignore", ".editorconfig", + ] + tracked = Set(split(readchomp(`git -C $(REPO_ROOT) ls-tree --name-only HEAD`), "\n")) + for entry in root_entries + @test entry in tracked + end + extras = sort(collect(setdiff(tracked, Set(root_entries)))) + if !isempty(extras) + println("root entries outside the allowlist: ", join(extras, ", ")) + end + @test isempty(extras) +end + +@testset "dependabot never batches a major" begin + # #124 grouped every frontend update, and with nothing requiring the + # checks it merged five framework majors (React 18->19, vite 6->8, + # TypeScript 5->7, react-router-dom 6->7, plotly 2->4) in one PR, with + # tsc already red until #140 repaired it. A major is a migration, not an + # update, so it must open its own PR: a Dependabot group only claims an + # update when its patterns match AND its update-types admit it -- a + # dependency matching no group gets its own PR. Every group must + # therefore NAME its update-types, and never "major". The frontend must + # also stay on the "bun" ecosystem: the 2026-09 security-update runs + # proved this account's npm_and_yarn updater refuses a Bun-managed + # project (see the header of .github/dependabot.yml). + dependabot = YAML.load_file(joinpath(REPO_ROOT, ".github", "dependabot.yml")) + @test dependabot["version"] == 2 + ecosystems = Dict{String,Any}() + for u in dependabot["updates"] + @test !haskey(ecosystems, u["package-ecosystem"]) + ecosystems[u["package-ecosystem"]] = u + end + @test sort(collect(keys(ecosystems))) == ["bun", "github-actions"] + @test ecosystems["bun"]["directory"] == "/frontend" + for u in dependabot["updates"] + for (_, group) in get(u, "groups", Dict{String,Any}()) + update_types = get(group, "update-types", nothing) + @test update_types !== nothing + @test !("major" in update_types) + end + end +end + +@testset "KYAML is the authoring dialect, ratcheted" begin + # YAML-POLICY Y-3 (standards, owner rulings D280/D307): KYAML is the + # estate's authoring dialect; a *.kyaml file that is not in KYAML form + # is a violation, an unconverted *.yml is migration backlog. The + # formatter is the standards repo's kyaml-format.sh, verbatim: the one + # rewriter the step-2 comment-preservation proof covers. Tool-named + # files (workflows, dependabot, codecov) keep their extensions; the + # shared config defaults convert as an upstream unit, not unilaterally. + formatter = joinpath(REPO_ROOT, "scripts", "kyaml-format.sh") + tracked = split(readchomp(`git -C $(REPO_ROOT) ls-files`), "\n") + kyamls = [joinpath(REPO_ROOT, f) for f in tracked if endswith(f, ".kyaml")] + for path in kyamls + proc = run(pipeline(ignorestatus(`bash $formatter --check $path`), + stdout = devnull, stderr = devnull)) + @test success(proc) + end +end diff --git a/test/unit/test_occupancy_grade.jl b/test/unit/test_occupancy_grade.jl new file mode 100644 index 00000000..8b94a0a5 --- /dev/null +++ b/test/unit/test_occupancy_grade.jl @@ -0,0 +1,270 @@ +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# Licensed under the GNU Affero General Public License version 3 (AGPLv3). + +## Grading a stage against the high-water-mark algebra (MMW-01, fork issue #105) +# The algebra is proved in occupancy-types' `OccupancyTypes.Reclaim`, over the HWM +# instance in resource-algebra; `scripts/fetch-siblings.sh` pins both and +# `scripts/check-occupancy.sh` type-checks what consumes them. Nothing here +# restates a sibling's definition: what is tested is the *reading* of a probe's +# samples as grades — that a task is sequenced from the previous one's drift, +# that the siblings' laws come out of that reading, and that the prediction +# reproduces the measurement the occupancy reading recorded on 2026-10-14. + +using MetaManifold.MemoryProbe +using OrderedCollections, YAML + +const MG = MetaManifold.MemoryProbe + +const RECORDED_OCCUPANCY = + normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "results-2026-10-14.yml")) +const RECORDED_SCHEDULES = + normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "schedules.yml")) + +# One task as the probe records it, with the sample window a schedule gives it. +# The windows determine which peaks remain live at each handover. +function _occ_task(peak; label = "t", first_sample = 1, last_sample = 2) + OrderedDict{String,Any}("label" => label, "processes" => 1, "samples" => 3, + "peak_rss_kib" => peak, "hwm_kib" => peak, + "first_sample" => first_sample, "last_sample" => last_sample) +end + +# One stage as `finish!` writes it: its tasks and the peak the samples saw. +_occ_stage(tasks; measured) = OrderedDict{String,Any}( + "probe" => MG.PROBE, "interval_ms" => 10, "samples" => 10, + "julia_allocated_bytes" => 1000, "julia_live_bytes_delta" => 500, + "child_peak_rss_kib" => measured, "tasks" => Any[tasks...]) + +@testset "occupancy grades" begin + @testset "a grade is (peak, drift), drift ≤ peak" begin + g = MG.Grade(90, 40) + @test g.peak == 90 + @test g.drift == 40 + @test MG.release(g) == 50 # what the frontier reclaims + # The invariant the HWM instance carries as its `bounded` field. + @test_throws ArgumentError MG.Grade(40, 90) + end + + @testset "sequencing starts the suffix from the drift (Reclaim.reclaim-law)" begin + a = MG.Grade(100, 40) + b = MG.Grade(60, 10) + s = MG.seq(a, b) + # peak (a ⊗ b) ≡ max (peak a) (drift a + peak b) + @test s.peak == max(a.peak, a.drift + b.peak) + @test s.drift == a.drift + b.drift + @test MG.seq(a, MG.NO_GRADE) == a # the unit holds nothing + @test s.peak <= a.peak + b.peak # Reclaim.occupancy≤cost + # Alternatives keep the larger of each component. + @test MG.alt(a, b) == MG.Grade(100, 40) + end + + @testset "the two shapes a stage has: serial and concurrent" begin + # One after the other: the first has exited (drift = 0), so the peak is + # the larger of the two and the cut reclaims the difference. + serial = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 2), + _occ_task(60; label = "otu", first_sample = 3, last_sample = 4)]; + measured = 100) + @test MG.predicted_peak_rss_kib(serial) == 100 + @test MG.cost_bound_rss_kib(serial) == 160 + @test MG.reclaimed_kib(serial) == 60 # Reclaim.reclaimed: strictly below cost + @test MG.disagreement(serial) === nothing + + # Together: the first is still resident when the second starts + # (drift = peak), so the peak is the sum and nothing is reclaimed. + together = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 4), + _occ_task(60; label = "otu", first_sample = 2, last_sample = 5)]; + measured = 160) + @test MG.predicted_peak_rss_kib(together) == 160 + @test MG.cost_bound_rss_kib(together) == 160 + @test MG.reclaimed_kib(together) == 0 # Reclaim.no-reclaim: drift = peak + @test MG.disagreement(together) === nothing + end + + @testset "overlap expires at each handover" begin + for (windows, predicted, remaining) in ( + ([(1, 2), (2, 3), (4, 5)], 160, 0), # an overlapping pair, then a serial task + ([(1, 3), (2, 5), (4, 6)], 160, 0), # pairwise overlap, never all three + ([(1, 6), (2, 3), (4, 5)], 180, 0), # the first survives a shorter task + ([(1, 5), (2, 5), (3, 5)], 240, 0), # genuine three-way overlap + ([(1, 2), (1, 2), (3, 4)], 160, 0), # children registered before sample 1 + ([(1, 0), (2, 3), (4, 0)], 180, 180), # unclosed windows remain live + ) + tasks = [_occ_task(peak; label = "t$i", first_sample = window[1], + last_sample = window[2]) + for (i, (peak, window)) in enumerate(zip((100, 60, 80), windows))] + occ = _occ_stage(tasks; measured = predicted) + grades = MG.task_grades(occ) + folded = foldl(MG.seq, grades) + @test folded.peak == predicted + @test folded.drift == remaining + @test MG.release(folded) == predicted - remaining + @test MG.cost_bound_rss_kib(occ) == 240 + @test MG.reclaimed_kib(occ) == 240 - predicted + @test MG.disagreement(occ) === nothing + @test MG.graded(occ)["predicted_peak_kib"] == predicted + # Record order does not change the sampled order. + @test MG.stage_grade(_occ_stage(reverse(tasks); measured = predicted)) == folded + end + @test !MG._overlaps((0, 0), (1, 2)) # a never-started task is not live + @test !MG._overlaps((1, 0), (0, 0)) + @test MG._overlaps((1, 2), (1, 2)) + @test MG._overlaps((1, 2), (2, 3)) # the end sample is included + @test !MG._overlaps((1, 2), (3, 4)) + end + + @testset "the reviewer's three-task example: expiry, not the sum of drifts" begin + # Windows [1,3], [2,10], [4,5] and peaks 100, 1, 100 KiB — CodeRabbit's + # example on #117. A overlaps B and exits before C starts, so the largest + # simultaneous sum is 101 (A+B at samples 2-3, then B+C at 4-5), not the + # 201 a rule that kept every earlier drift would read, and not the 201 KiB + # cost. A's release at C's boundary makes B's delta negative, which is how + # the fold stops charging for it. + tasks = [_occ_task(100; label = "A", first_sample = 1, last_sample = 3), + _occ_task(1; label = "B", first_sample = 2, last_sample = 10), + _occ_task(100; label = "C", first_sample = 4, last_sample = 5)] + occ = _occ_stage(tasks; measured = 101) + @test [g.drift for g in MG.task_grades(occ)] == [100, -99, -1] + @test MG.predicted_peak_rss_kib(occ) == 101 + @test MG.cost_bound_rss_kib(occ) == 201 + @test MG.reclaimed_kib(occ) == 100 # the 100 KiB that never coexist + @test MG.disagreement(occ) === nothing + # The bound the expiry sharpens: 201 KiB resident at once is now the + # disagreement, where before the fix it was the prediction. + @test MG.disagreement(_occ_stage(tasks; measured = 201)) isa String + end + + @testset "registration keeps the first child's sample boundary" begin + # No background sampler: watch! takes the first sample deterministically. + probe = MG.Probe(0.01, ReentrantLock(), MG.TaskTrace[], MG.TaskTrace("(stage)"), + 0, 0, Base.gc_num(), Base.gc_live_bytes(), false, nothing, + nothing, nothing, nothing, Vector{UInt8}(undef, 8192), + Vector{UInt8}(undef, 64)) + function child!() + proc = run(`sleep 0.01`; wait = false) + pid = MG.watch!(proc) + try + wait(proc) + finally + MG.unwatch!(pid) + end + end + MG.within(probe) do + MG.task("two children") do + child!() + @test probe.samples == 1 + @test probe.tasks[1].first_sample == 1 + @test probe.tasks[1].last_sample == 1 + MG.task("between children") do + child!() + end + child!() + end + end + first_task, between = probe.tasks + @test first_task.processes == 2 + @test first_task.first_sample == 1 + @test first_task.last_sample == 3 + @test between.first_sample == 2 + @test MG._overlaps((first_task.first_sample, first_task.last_sample), + (between.first_sample, between.last_sample)) + @test MG.TaskTrace("never started").first_sample == 0 + end + + @testset "the graded record says what it is" begin + rec = _occ_stage([_occ_task(100; label = "vsearch:asv", first_sample = 1, last_sample = 4), + _occ_task(60; label = "vsearch:otu", first_sample = 2, last_sample = 5)]; + measured = 160) + g = MG.graded(rec) + @test g["algebra"] == "ResourceAlgebra.Instances.HWM" + @test g["predicted_peak_kib"] == 160 + @test g["cost_bound_kib"] == 160 + @test g["reclaimed_kib"] == 0 + @test g["measured_peak_kib"] == 160 + @test g["disagreement"] === nothing + @test [t["label"] for t in g["tasks"]] == ["vsearch:asv", "vsearch:otu"] + @test g["tasks"][1]["drift_kib"] == 100 # still resident at the handover + @test g["tasks"][2]["drift_kib"] == -100 # the earlier task also exited + @test g["tasks"][2]["release_kib"] == 160 # both peaks are reclaimed + @test g["drift_kib"] == 0 + end + + @testset "a measured peak above the prediction is a disagreement" begin + # The prediction is an upper bound, so this is the one contradiction it + # allows: a sample saw more than the timed timeline can hold. + rec = _occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; measured = 250) + d = MG.disagreement(rec) + @test d isa String + @test occursin("250", d) && occursin("100", d) + # At or below the bound there is nothing to report. + @test MG.disagreement(_occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; + measured = 100)) === nothing + end + + @testset "a record with no sample windows is not graded" begin + # Records written before the windows were sampled cannot be read as a + # timeline, so nothing is predicted for them rather than an order assumed. + rec = OrderedDict{String,Any}("child_peak_rss_kib" => 100, + "tasks" => Any[OrderedDict{String,Any}("label" => "t", "peak_rss_kib" => 100)]) + @test MG.predicted_peak_rss_kib(rec) === nothing + @test MG.graded(rec) === nothing + @test MG.disagreement(rec) === nothing # not measured is not a disagreement + end + + @testset "finish! grades the record it writes" begin + # A record the probe writes itself carries its tasks' sample windows, so + # `finish!` attaches the grades read from them; the run with real vsearch + # children is test/occupancy_disagreement_test.jl. + if !Sys.islinux() + @info "occupancy grades: not Linux, so there is no /proc to sample; the finish! grading check is SKIPPED" + else + probe = MG.start(Dict("occupancy_probe" => Dict("enabled" => true, + "interval_ms" => 10))) + MG.within(probe) do + MG.task("sleep") do + proc = run(`sleep 0.2`; wait = false) + pids = MG.watch!(proc) + try + wait(proc) + finally + MG.unwatch!(pids) + end + end + end + rec = MG.finish!(probe) + @test length(rec["tasks"]) == 1 # one task; no stage trace + t = rec["tasks"][1] + @test t["label"] == "sleep" && t["samples"] > 0 + @test t["peak_rss_kib"] > 0 # sampled from /proc + @test t["last_sample"] >= t["first_sample"] # the window watch!/unwatch! set + @test haskey(rec, "grades") + @test rec["grades"]["predicted_peak_kib"] == MG.predicted_peak_rss_kib(rec) + @test rec["grades"]["measured_peak_kib"] == rec["child_peak_rss_kib"] + @test MG.disagreement(rec) === nothing + end + end + + @testset "the prediction reproduces the recorded measurement" begin + # docs/occupancy/results-2026-10-14.yml is the probe's own record of the + # run docs/occupancy-2026-10.adoc publishes. The schedules are the ones the + # .sir files in docs/occupancy state — serial for two orders, together for + # the third — and the peaks are the recorded ones: nothing here is invented. + orders = YAML.load_file(RECORDED_OCCUPANCY)["orders"] + # The interleavings the run used, from the same file the gate reads + # (scripts/check-occupancy-grades.jl), so the two cannot disagree. + schedules = YAML.load_file(RECORDED_SCHEDULES) + for (name, rec) in orders + windows = schedules[name] + tasks = [_occ_task(t["peak_rss_kib"]; label = t["label"], + first_sample = windows[i][1], last_sample = windows[i][2]) + for (i, t) in enumerate(rec["tasks"])] + occ = _occ_stage(tasks; measured = rec["child_peak_rss_kib"]) + graded = MG.graded(occ) + @test MG.predicted_peak_rss_kib(occ) == rec["child_peak_rss_kib"] + @test MG.disagreement(occ) === nothing + peaks = [t["peak_rss_kib"] for t in rec["tasks"]] + # Serial: the larger peak. Together: both resident, so nothing is reclaimed. + @test graded["reclaimed_kib"] == + (name == "concurrent" ? 0 : sum(peaks) - maximum(peaks)) + @test graded["cost_bound_kib"] == sum(peaks) + end + end +end diff --git a/test/unit/test_server_security.jl b/test/unit/test_server_security.jl index 2bb7086d..9ef2fad6 100644 --- a/test/unit/test_server_security.jl +++ b/test/unit/test_server_security.jl @@ -34,6 +34,14 @@ end SV.ServerState.set_root!(tmp) SV._bound_host[] = "127.0.0.1" SV._bound_port[] = 8080 + # The baseline assertions below hold only with no extra origins + # configured. JULIA_METAMANIFOLD_ALLOWED_ORIGINS is a supported + # deployment setting (split deployments, tunnels), and a developer + # running the suite with it set would otherwise fail them, so it is + # cleared here and restored in the finally. + saved_origins = get(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS", nothing) + haskey(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") && + delete!(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") mkpath(joinpath(tmp, "data", "StudyS", "run1")) touch(joinpath(tmp, "data", "StudyS", "run1", "s1_R1.fastq.gz")) mkpath(joinpath(tmp, "projects", "StudyS", "run1")) @@ -77,6 +85,43 @@ end @test _ss_request("GET", studies, ["Host" => "evil.example:8080"]).status == 403 end + @testset "Wildcard bind serves, and admits only, the reached IP" begin + studies = "/api/v1/studies" + SV._bound_host[] = "0.0.0.0" + lan = "Host" => "192.168.1.10:8080" + ui = "Origin" => "http://192.168.1.10:8080" + # The UI served at the machine's address: its origin is admitted, + # with CORS headers, a preflight answer, and writes that reach the + # route (this one fails in the route itself, not in the guard). + r = _ss_request("GET", studies, [lan, ui]) + @test r.status == 200 + @test SV.HTTP.header(r, "Access-Control-Allow-Origin") == "http://192.168.1.10:8080" + @test _ss_request("OPTIONS", studies, [lan, ui]).status == 204 + @test _ss_request("POST", "/api/v1/studies/StudyS/report?kind=bogus&ext=.txt", + [lan, ui, "Content-Type" => "text/plain"], "x").status == 400 + @test !isdir(joinpath(tmp, "projects", "StudyS", "report")) + # An IPv6 literal that matches its Host is admitted the same way. + @test _ss_request("GET", studies, + ["Host" => "[fd00::10]:8080", "Origin" => "http://[fd00::10]:8080"]).status == 200 + # Another address, another port, or another scheme: still refused. + for o in ("http://192.168.1.11:8080", "http://192.168.1.10:9090", + "https://192.168.1.10:8080") + @test _ss_request("GET", studies, [lan, "Origin" => o]).status == 403 + end + # A DNS-rebinding page: the domain matches its own Host, but a + # name is not an IP literal, so it is still refused. + @test _ss_request("GET", studies, + ["Host" => "evil.example:8080", "Origin" => "http://evil.example:8080"]).status == 403 + # A hostname the machine is reached by still has to be listed. + withenv("JULIA_METAMANIFOLD_ALLOWED_ORIGINS" => "http://lab-box:8080") do + @test _ss_request("GET", studies, + ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 200 + end + @test _ss_request("GET", studies, + ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 403 + SV._bound_host[] = "127.0.0.1" + end + @testset "Oxygen docs and metrics are not served" begin @test SV._SERVE_OPTIONS.docs === false @test SV._SERVE_OPTIONS.metrics === false @@ -121,6 +166,8 @@ end @test SV.HTTP.header(f, "Content-Security-Policy") == "" end finally + saved_origins === nothing || + (ENV["JULIA_METAMANIFOLD_ALLOWED_ORIGINS"] = saved_origins) SV.ServerState._root[] = old_root SV._bound_host[], SV._bound_port[] = old_host, old_port rm(tmp; recursive=true, force=true)