From bf68ba1639917714027fb9c38088c62e6bb32dc4 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:45:56 +0100 Subject: [PATCH 01/45] feat(pipeline): record stage memory as occupancy grades and run the two-thread disagreement test (#106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MMW-01, part 1 of 2: the probe and the test. This is the unit that would go upstream to JoshuaJewell once the fork is re-synced. Part 2, the dated typed reading, is stacked on this branch. ## Why The vsearch stage runs its ASV and OTU searches concurrently in a full run, and one after the other in a single-stage re-run. Nothing measures what either shape occupies. The searches are child processes, so `Sys.maxrss()` of the Julia process never sees them. The high-water-mark algebra (occupancy-types, choreographic-types A-2/A-3) makes a falsifiable prediction about this shape: - the cost is the same in every order; - the memory peak is not: the larger search's peak serially, up to the sum concurrently. ## What changes **`MemoryProbe`** (`src/core/memory_probe.jl`). Off unless a `pipeline.yml` sets `occupancy_probe.enabled: true`. For each attested stage it records under a new optional `occupancy` key: - `julia_allocated_bytes`: the `Base.gc_num()` delta over the stage; - per task: the largest sampled VmRSS of the children it started (`peak_rss_kib`) and their kernel VmHWM (`hwm_kib`); - `child_peak_rss_kib`: the largest sum of the stage's children's VmRSS at one sampling instant. It samples `/proc` every 10 ms (`interval_ms`) with preallocated buffers, so the sampling barely touches the allocation figure it reports. **Hooks.** - `Tools._run_killable` registers each child. - `_attest` wraps the stage and passes the record to `Provenance.record_stage!(...; occupancy)`. - Both vsearch call sites label their searches `vsearch:asv` and `vsearch:otu`. **With the flag off nothing changes.** `start` returns `nothing` and every hook passes straight through, so a stage record has exactly the keys it had before. The key is deliberately *not* added to `config/defaults/pipeline.yml`, only documented there as a comment: the run config is embedded in `attestation.yml`, so a default key would change every run's record even with the flag off. **Robustness.** The probe never throws into the stage it measures. `getpid` is read once, while the child is alive. ## The disagreement test `test/occupancy_disagreement_test.jl` runs with the full suite. It runs the two searches in three orders, ASV then OTU, OTU then ASV, and concurrent, on a generated fixture of 1000 reference sequences × 400 bp with 300 ASV and 150 OTU queries. **Why that fixture:** of a calibrated ladder (250, 500, 1000, 2000), it is the smallest that kept both searches resident together for at least 15 samples in all of five repeats. **Asserted:** - the cost agrees within 1%; - the hits are byte-identical in every order. **Recorded, not asserted:** each search's peak and the stage peak. Locally: | order | ASV peak | OTU peak | stage peak (KiB) | Julia allocation (bytes) | |---|---|---|---|---| | ASV then OTU | 9204 | 9180 | 9204 | 978432 | | OTU then ASV | 9152 | 9192 | 9192 | 978720 | | concurrent | 9176 | 9204 | **18380** | 978816 | The cost spread is 0.04%. The concurrent peak is twice the serial one. The prediction would have been falsified had the concurrent peak matched the serial one; it did not. **One finding, not changed here.** The test runs vsearch with `--threads 1`. With several threads, vsearch's output order varies from run to run, and `_sort_hits_by_query!` then does a different amount of work, because its `by=` key splits each line on every comparison. That moved the stage's allocation by up to 12% between repeats of the *same* order. Precomputing the sort key would make production cost order-independent too. The file also covers: - the flag off and on through the real `_attest`; - a failed stage, which keeps its occupancy and still rethrows; - old records, which still parse and merge; - a 64 MiB child against a `sleep` control; - chained commands and an instantly exiting child. ## Verification Run locally with Julia 1.12.5 `-t 2` and vsearch 2.30.5 (the pinned version): - `julia --project=. -t 2 test/occupancy_disagreement_test.jl`: 46/46. - `test/runtests.jl provenance tools routes jobs config_hashing`: 547 passed, 10 skipped (`@test_skip`). One error: the live R probe, which needs the dada2 R package that CI installs and this machine lacks. - Five planted mutants are each caught by a failing test: - children ignored; - flag ignored; - key written with the flag off; - stage peak computed as the sum of the task peaks; - extra work in one order (cost spread 0.249). ## What this leaves out - Typed peaks, Session IR and the dated table: part 2, stacked on this branch. - The open bridge that the typed peak bounds the measured peak: occupancy-types, OPEN. - Memory on the R side. - Stages offloaded with `remote:`, where the local child is an ssh client. - Non-Linux hosts. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_015pMjy981ie74uCttyYDanC --- _Generated by [Claude Code](https://claude.ai/code/session_015pMjy981ie74uCttyYDanC)_ Co-authored-by: Claude Opus 5.5 --- config/defaults/pipeline.yml | 15 ++ src/MetaManifold.jl | 2 + src/core/memory_probe.jl | 343 ++++++++++++++++++++++++++++ src/core/provenance.jl | 10 +- src/pipeline/tools.jl | 6 + src/server/routes/pipeline.jl | 34 ++- test/occupancy_disagreement_test.jl | 320 ++++++++++++++++++++++++++ test/runtests.jl | 6 + 8 files changed, 727 insertions(+), 9 deletions(-) create mode 100644 src/core/memory_probe.jl create mode 100644 test/occupancy_disagreement_test.jl diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 11198e7a..6f50ed68 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -354,3 +354,18 @@ analysis: # Fraction of samples, 0 to 1, in which a taxon must have reads to be # tested. Taxa below it are listed as filtered, without a p-value. min_prevalence: 0.0 + +## Occupancy probe (MMW-01) +# Measures each attested stage's memory: Julia's allocation total, and the +# sampled resident memory of the tools it runs as child processes (Linux /proc). +# Recorded under the stage's `occupancy` key in attestation.yml. See the +# header of src/core/memory_probe.jl. +# +# Off unless set, and deliberately left commented out here: the run config is +# embedded in attestation.yml, so a default key would change every run's record +# even with the probe off. Set it by editing a pipeline.yml (the config editor +# only offers keys that have a factory default). +# occupancy_probe: +# enabled: true +# # Sampling interval in milliseconds (at least 1). +# interval_ms: 10 diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index e3c70714..31db2a5c 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -5,6 +5,8 @@ include("core/types.jl") include("core/r_runtime.jl") # Provenance probes R through the shared runtime lock, so it follows r_runtime.jl. include("core/provenance.jl") +# Tools registers the children it starts with the probe, so it precedes pipeline/. +include("core/memory_probe.jl") include("core/log.jl") include("core/config.jl") include("core/databases.jl") diff --git a/src/core/memory_probe.jl b/src/core/memory_probe.jl new file mode 100644 index 00000000..6406c37a --- /dev/null +++ b/src/core/memory_probe.jl @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) + +module MemoryProbe + +## Occupancy probe +# Measures what an attested stage occupied, for the stage's Attestation record: +# +# * Julia-side allocation, as the `Base.gc_num()` delta over the stage. This is +# a cost: it only ever adds up, whatever order the stage's work ran in. +# * Each child process's resident memory, sampled from /proc. The tools run as +# child processes (vsearch, swarm, cd-hit, ...), so `Sys.maxrss()` of this +# process never sees them. +# +# Two figures come from the samples. A task's `peak_rss_kib` is the largest VmRSS +# sampled for the processes that task started. The stage's `child_peak_rss_kib` is +# the largest SUM of VmRSS over every process of the stage at one sampling +# instant. For two tasks run one after the other it is the larger of their peaks; +# for two that run concurrently it can approach their sum. That order dependence, +# invisible in every output file, is what the measurement exists to show. +# +# `hwm_kib` is the kernel's own high-water mark (VmHWM) for a task's processes, as +# last read before they exited. It is exact up to the last sample, so it is at +# least `peak_rss_kib`; growth after the last sample is not seen. A process that +# lives for less than one interval may not be sampled at all, which `samples: 0` +# reports rather than a misleading zero peak. +# +# The probe is off unless the run config sets `occupancy_probe.enabled: true`. +# Off, `start` returns `nothing` and every other entry point is a pass-through, +# so the stage records and the outputs are exactly what they were without it. +# +# Linux only (/proc). Elsewhere, and for stages offloaded with `remote:` (where +# the local child is an ssh client), the child figures do not describe the work. +# The allocation figure is process-wide: while the study loop runs several runs' +# pipelines at once, a stage's figure includes the others' allocations too. The +# child figures are not affected, since each stage samples only its own children. + +using Base.ScopedValues: ScopedValue, with +using OrderedCollections: OrderedDict + +const PROBE = "MemoryProbe 1" +const DEFAULT_INTERVAL_MS = 10 + +# One labelled unit of a stage's work (one spawned search, say) and the child +# processes it started. +mutable struct TaskTrace + label :: String + live :: Vector{Int} # pids started and not yet waited for + processes :: Int # every child started under this label + samples :: Int # samples that found at least one live process + peak_kib :: Int + hwm_kib :: Int +end +TaskTrace(label::AbstractString) = TaskTrace(String(label), Int[], 0, 0, 0, 0) + +mutable struct Probe + interval :: Float64 + lock :: ReentrantLock + tasks :: Vector{TaskTrace} + stage :: TaskTrace # children started outside any `task` + samples :: Int + peak_kib :: Int + gc_before :: Base.GC_Num + live_before :: Int + running :: Bool + sampler :: Union{Task,Nothing} + timer :: Union{Timer,Nothing} + record :: Union{OrderedDict{String,Any},Nothing} + error :: Union{String,Nothing} # why sampling stopped early, if it did + buf :: Vector{UInt8} # /proc reads; preallocated so a sample + path :: Vector{UInt8} # allocates nothing that would count as work +end + +const STAGE = ScopedValue{Union{Probe,Nothing}}(nothing) +const TASK = ScopedValue{Union{TaskTrace,Nothing}}(nothing) + +## Configuration +_section(cfg) = cfg isa AbstractDict ? get(cfg, "occupancy_probe", nothing) : nothing + +""" + enabled(run_cfg) -> Bool + +Whether the run config turns the probe on: `occupancy_probe.enabled: true`. Any +other value, or no `occupancy_probe` section at all, leaves it off. +""" +function enabled(cfg) + s = _section(cfg) + s isa AbstractDict && get(s, "enabled", false) === true +end + +""" + interval_seconds(run_cfg) -> Float64 + +The sampling interval, from `occupancy_probe.interval_ms` (default 10 ms, at +least 1 ms). +""" +function interval_seconds(cfg) + s = _section(cfg) + ms = s isa AbstractDict ? get(s, "interval_ms", DEFAULT_INTERVAL_MS) : DEFAULT_INTERVAL_MS + (ms isa Real && !(ms isa Bool) && ms >= 1) || (ms = DEFAULT_INTERVAL_MS) + Float64(ms) / 1000 +end + +## Reading /proc without allocating +# "/proc//status" into `path`, NUL-terminated. +function _status_path!(path::Vector{UInt8}, pid::Int) + k = 0 + for c in codeunits("/proc/"); k += 1; path[k] = c; end + nd = ndigits(pid); d = pid + for i in nd:-1:1 + path[k + i] = UInt8('0') + UInt8(d % 10); d ÷= 10 + end + k += nd + for c in codeunits("/status"); k += 1; path[k] = c; end + path[k + 1] = 0x00 + nothing +end + +# Bytes of /proc//status read into `buf`, or -1 if the process is gone. +function _read_status!(buf::Vector{UInt8}, path::Vector{UInt8}, pid::Int) + _status_path!(path, pid) + fd = GC.@preserve path ccall(:open, Cint, (Ptr{UInt8}, Cint), path, 0) + fd < 0 && return -1 + total = 0 + GC.@preserve buf while total < length(buf) + r = ccall(:read, Cssize_t, (Cint, Ptr{UInt8}, Csize_t), + fd, pointer(buf, total + 1), length(buf) - total) + r <= 0 && break + total += r + end + ccall(:close, Cint, (Cint,), fd) + total +end + +# The kB value of the line starting `key` (6 bytes, e.g. "VmRSS:"), or -1. +function _field_kib(buf::Vector{UInt8}, n::Int, key::NTuple{6,UInt8}) + i = 1 + while i <= n - 6 + if (i == 1 || buf[i - 1] == UInt8('\n')) && + buf[i] == key[1] && buf[i + 1] == key[2] && buf[i + 2] == key[3] && + buf[i + 3] == key[4] && buf[i + 4] == key[5] && buf[i + 5] == key[6] + j = i + 6 + while j <= n && (buf[j] == UInt8(' ') || buf[j] == UInt8('\t')); j += 1; end + v = 0 + while j <= n && UInt8('0') <= buf[j] <= UInt8('9') + v = 10v + Int(buf[j] - UInt8('0')); j += 1 + end + return v + end + i += 1 + end + -1 +end + +const VMRSS = Tuple(codeunits("VmRSS:")) +const VMHWM = Tuple(codeunits("VmHWM:")) + +## Sampling +# One sample: each live child's VmRSS and VmHWM, summed per task and over the stage. +function _sample!(p::Probe) + lock(p.lock) do + total = 0 + for t in p.tasks + total += _sample_task!(p, t) + end + total += _sample_task!(p, p.stage) + p.samples += 1 + p.peak_kib = max(p.peak_kib, total) + end + nothing +end + +function _sample_task!(p::Probe, t::TaskTrace) + isempty(t.live) && return 0 + rss = 0; seen = false + for pid in t.live + n = _read_status!(p.buf, p.path, pid) + n > 0 || continue + r = _field_kib(p.buf, n, VMRSS) + r < 0 && continue # a zombie has no memory lines + seen = true + rss += r + t.hwm_kib = max(t.hwm_kib, _field_kib(p.buf, n, VMHWM)) + end + seen || return 0 + t.samples += 1 + t.peak_kib = max(t.peak_kib, rss) + rss +end + +""" + start(run_cfg) -> Union{Probe,Nothing} + +A running probe for one stage, or `nothing` when the run config does not enable +it. Pass the result to `within` and then `finish!`. +""" +function start(cfg) + enabled(cfg) || return nothing + p = Probe(interval_seconds(cfg), ReentrantLock(), TaskTrace[], TaskTrace("(stage)"), + 0, 0, Base.gc_num(), Base.gc_live_bytes(), true, nothing, nothing, nothing, + nothing, Vector{UInt8}(undef, 8192), Vector{UInt8}(undef, 64)) + p.timer = Timer(0; interval = p.interval) + timer = p.timer + # A probe must never fail the stage it measures: a sampling error ends the + # sampling and is reported in the record instead. + p.sampler = Threads.@spawn begin + try + while p.running + _sample!(p) + isopen(timer) || break + wait(timer) + end + catch err + (err isa EOFError && !p.running) || (p.error = sprint(showerror, err)) + end + end + p +end + +""" + within(f, probe) + +Run `f()` with `probe` as the current stage's probe, so the children it starts +(directly or from tasks it spawns) are sampled. With `probe === nothing` this is +just `f()`. +""" +within(f, ::Nothing) = f() +within(f, p::Probe) = with(f, STAGE => p) + +""" + task(f, label) + +Run `f()` as one labelled unit of the current stage. Children it starts are +reported under `label`. Outside a probed stage this is just `f()`. +""" +function task(f, label::AbstractString) + p = STAGE[] + p === nothing && return f() + t = TaskTrace(label) + lock(() -> push!(p.tasks, t), p.lock) + with(f, TASK => t) +end + +# The trace a child started now belongs to, or nothing when nothing is probed. +function _current() + p = STAGE[] + p === nothing && return nothing, nothing + t = TASK[] + p, something(t, p.stage) +end + +""" + watch!(process) -> Union{Int,Nothing} + +Register a just-started child process with the current stage's probe, sample it +at once, and return its pid for `unwatch!`. Called by `Tools._run_killable`. +Returns `nothing`, having done nothing, when the stage is not probed. A child that +has already exited is counted but cannot be sampled. Never throws: a probe must +not fail the stage it measures. +""" +function watch!(proc::Base.Process) + p, t = _current() + p === nothing && return nothing + # `getpid` throws once the child has exited, so the pid is read now or never. + pid = try + Int(getpid(proc)) + catch + nothing + end + lock(p.lock) do + t.processes += 1 + pid === nothing || push!(t.live, pid) + end + pid === nothing || _sample!(p) + pid +end + +# A pipeline of commands: each of its processes, as one task's children. +function watch!(chain::Base.ProcessChain) + pids = Int[] + for proc in chain.processes + pid = watch!(proc) + pid === nothing || push!(pids, pid) + end + isempty(pids) ? nothing : pids +end + +# Anything else `run` can return is not sampled. +watch!(_) = nothing + +""" + unwatch!(pid) + +Stop sampling a child that has been waited for (the pid `watch!` returned), so +the pid cannot be mistaken for a later, unrelated process. A no-op for `nothing`. +""" +unwatch!(::Nothing) = nothing +unwatch!(pids::Vector{Int}) = foreach(unwatch!, pids) +function unwatch!(pid::Int) + p, t = _current() + p === nothing && return nothing + lock(p.lock) do + filter!(!=(pid), t.live) + end + nothing +end + +_task_record(t::TaskTrace) = OrderedDict{String,Any}( + "label" => t.label, + "processes" => t.processes, + "samples" => t.samples, + "peak_rss_kib" => t.peak_kib, + "hwm_kib" => t.hwm_kib, +) + +""" + finish!(probe) -> Union{OrderedDict,Nothing} + +Stop sampling and return the stage's `occupancy` record (the same record on every +later call). `nothing` for `nothing`. +""" +finish!(::Nothing) = nothing +function finish!(p::Probe) + p.record === nothing || return p.record + p.running = false + close(p.timer) + wait(p.sampler) + allocated = Base.GC_Diff(Base.gc_num(), p.gc_before).allocd + traces = [p.tasks; p.stage.processes > 0 ? [p.stage] : TaskTrace[]] + p.record = OrderedDict{String,Any}( + "probe" => PROBE, + "interval_ms" => round(Int, p.interval * 1000), + "samples" => p.samples, + "julia_allocated_bytes" => allocated, + "julia_live_bytes_delta" => Base.gc_live_bytes() - p.live_before, + "child_peak_rss_kib" => p.peak_kib, + "tasks" => Any[_task_record(t) for t in traces], + ) + p.error === nothing || (p.record["error"] = p.error) + p.record +end + +end # module MemoryProbe diff --git a/src/core/provenance.jl b/src/core/provenance.jl index aa232c06..b28912d4 100644 --- a/src/core/provenance.jl +++ b/src/core/provenance.jl @@ -818,11 +818,15 @@ _stage_index(att::Attestation, name::AbstractString) = findfirst(s -> get(s, "name", nothing) == String(name), stages(att)) """ - record_stage!(att, name, env; started, finished, status, commands, outputs) -> Attestation + record_stage!(att, name, env; started, finished, status, commands, outputs, occupancy) -> Attestation Record one stage's own captured environment, replacing any earlier record of that stage. +`occupancy` is the stage's `MemoryProbe` record. It is stored under the stage's +`occupancy` key only when given, so a stage recorded without the probe has +exactly the keys it always had. + Re-running a single stage of an existing run must update that stage alone, because a run directory is an accretion and the other stages' records remain true. The run-level summary is re-derived on every call. @@ -832,7 +836,8 @@ function record_stage!(att::Attestation, name::AbstractString, env::CapturedEnvi finished::AbstractString = timestamp(), status::AbstractString = "ok", commands::AbstractVector = String[], - outputs::AbstractVector = OrderedDict{String,Any}[]) + outputs::AbstractVector = OrderedDict{String,Any}[], + occupancy::Union{AbstractDict,Nothing} = nothing) stage = OrderedDict{String,Any}( "name" => String(name), "started" => String(started), @@ -847,6 +852,7 @@ function record_stage!(att::Attestation, name::AbstractString, env::CapturedEnvi "outputs" => Any[OrderedDict{String,Any}(string(k) => v for (k, v) in o) for o in outputs], ) + isnothing(occupancy) || (stage["occupancy"] = occupancy) index = _stage_index(att, name) isnothing(index) ? push!(stages(att), stage) : (stages(att)[index] = stage) return summarise!(att) diff --git a/src/pipeline/tools.jl b/src/pipeline/tools.jl index d8e12062..ac212228 100644 --- a/src/pipeline/tools.jl +++ b/src/pipeline/tools.jl @@ -14,6 +14,7 @@ export cutadapt, vsearch, multiqc, cdhit, tool_bin, _sq, _run_logged, _run_killa using ..PipelineTypes using ..PipelineLog using ..Config + import ..MemoryProbe _sq(s::AbstractString) = "'" * replace(s, "'" => "'\\''") * "'" @@ -24,13 +25,18 @@ export cutadapt, vsearch, multiqc, cdhit, tool_bin, _sq, _run_logged, _run_killa # Truncation is the stage's own affair, done once at entry via reset_tool_logs. # Run a command and wait for it. If the waiting task is interrupted (a job # cancel), the child process is killed so it cannot keep writing outputs. + # When the stage is probed (occupancy_probe.enabled), the child's memory is + # sampled while it runs; otherwise watch!/unwatch! do nothing. function _run_killable(cmd::Base.AbstractCmd) p = run(cmd; wait=false) + watched = MemoryProbe.watch!(p) try wait(p) catch process_running(p) && kill(p) rethrow() + finally + MemoryProbe.unwatch!(watched) end success(p) || throw(ProcessFailedException(p)) nothing diff --git a/src/server/routes/pipeline.jl b/src/server/routes/pipeline.jl index 306d8c1a..ee1bdedd 100644 --- a/src/server/routes/pipeline.jl +++ b/src/server/routes/pipeline.jl @@ -9,6 +9,7 @@ using JSON3, RCall using ..RRuntime: with_r_lock const Provenance = MetaManifold.Provenance +const MemoryProbe = MetaManifold.MemoryProbe ## Tagging helpers @@ -180,17 +181,26 @@ end # # A stage that skipped leaves its earlier record alone: writing the current # environment over it would attribute old outputs to new tool versions. +# +# With `occupancy_probe.enabled: true` in the run config, the stage's memory is +# measured (MemoryProbe) and recorded under the stage's `occupancy` key. Off, the +# probe is `nothing` and the record is exactly what it was without it. function _attest(f::Function, att, env, name::AbstractString; config_path::String) started = Provenance.timestamp() ran_before = stage_run_count(config_path) + probe = MemoryProbe.start(get(get(att.doc, "config", Dict()), "run_config", nothing)) try - result = f() + result = MemoryProbe.within(f, probe) stage_run_count(config_path) > ran_before && - Provenance.record_stage!(att, name, env; started, outputs=_stage_outputs(result)) + Provenance.record_stage!(att, name, env; started, outputs=_stage_outputs(result), + occupancy=MemoryProbe.finish!(probe)) return result catch - Provenance.record_stage!(att, name, env; started, status="failed") + Provenance.record_stage!(att, name, env; started, status="failed", + occupancy=MemoryProbe.finish!(probe)) rethrow() + finally + MemoryProbe.finish!(probe) end end @@ -261,9 +271,13 @@ function _run_full_pipeline(study::String, run::String, db_config::String, # OTU search it claims to cover. asv_tax, otu_tax = if vsearch_enabled _attest(att, env, "vsearch"; config_path) do - asv_task = Threads.@spawn(vsearch(project, asvs, dbs["$(db_name)_vsearch"])) + asv_task = Threads.@spawn MemoryProbe.task("vsearch:asv") do + vsearch(project, asvs, dbs["$(db_name)_vsearch"]) + end otu_task = (swarm_enabled && !isnothing(otus)) ? - Threads.@spawn(vsearch(project, otus, dbs["$(db_name)_vsearch"])) : nothing + Threads.@spawn(MemoryProbe.task("vsearch:otu") do + vsearch(project, otus, dbs["$(db_name)_vsearch"]) + end) : nothing # Wait for both before rethrowing, so a failed ASV search cannot leave the OTU search running. tasks = filter(!isnothing, Any[asv_task, otu_task]) foreach(t -> (try wait(t) catch end), tasks) @@ -438,12 +452,18 @@ function _run_stage(study::String, run::String, stage::String, end elseif step == "vsearch" if vsearch_enabled + # Serial here, concurrent in the full pipeline: the same two + # searches in the two shapes the occupancy probe distinguishes. _record("vsearch") do asvs = _asvresult_from_disk(run_dir) - vsearch(project, asvs, dbs["$(db_name)_vsearch"]) + MemoryProbe.task("vsearch:asv") do + vsearch(project, asvs, dbs["$(db_name)_vsearch"]) + end otus = _oturesult_from_disk(run_dir) if !isnothing(otus) && swarm_enabled - vsearch(project, otus, dbs["$(db_name)_vsearch"]) + MemoryProbe.task("vsearch:otu") do + vsearch(project, otus, dbs["$(db_name)_vsearch"]) + end end end end diff --git a/test/occupancy_disagreement_test.jl b/test/occupancy_disagreement_test.jl new file mode 100644 index 00000000..b609ed38 --- /dev/null +++ b/test/occupancy_disagreement_test.jl @@ -0,0 +1,320 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# MMW-01: the occupancy probe, and the two-thread disagreement test. +# +# The vsearch stage runs two independent searches, ASV and OTU, concurrently in +# the full pipeline and one after the other in a single-stage re-run. The +# high-water-mark algebra (occupancy-types, resource-algebra `Instances/HWM`) +# predicts: +# +# * cost commutes: the work done, measured as Julia's allocation total for the +# stage, is the same in every order (choreographic-types A-2). This is +# ASSERTED, and it can fail. +# * each search's own peak is the same in every order; +# * the stage's peak does not commute: serially it is the larger of the two +# searches' peaks (each child exits holding nothing, so its drift is 0), while +# concurrently it can approach their sum (A-3). These are RECORDED in the test +# log, not asserted. +# +# If the stage peak came out the same in every order, the prediction would be +# falsified for this workload: the searches' memory would not overlap. +# +# The fixture is the smallest of a calibrated ladder (250, 500, 1000, 2000 +# reference sequences of 400 bp; 300 ASV and 150 OTU queries) for which, in five +# repeats each on a 4-core host, both searches were resident together for at +# least 15 samples at 10 ms and the concurrent stage peak was at least 1.9 times +# the larger search's: 1000 sequences (17-20 samples, 2.0 times). 250 and 500 +# also overlap, by 12-18 samples, too close to the edge for a slower runner. It is +# generated here, deterministically, so no data is checked in. +# +# vsearch runs with --threads 1 here. With several threads it writes its hits in +# whatever order its threads finish, and `_sort_hits_by_query!` then does a +# different amount of work (its sort key splits each line on every comparison), +# which moved the stage's allocation total by up to 12% between repeats of the +# SAME order. On one thread the output order is fixed, and the three orders +# agreed to within 0.08% over five repeats each. Run directly with +# julia --project=. -t 2 test/occupancy_disagreement_test.jl +# and set OCCUPANCY_RESULTS= to also write the numbers as YAML. + +using Test, YAML, OrderedCollections +using MetaManifold + +const MP = MetaManifold.MemoryProbe +const PV = MetaManifold.Provenance +const TL = MetaManifold.Tools + +const PROBE_ON = Dict("occupancy_probe" => Dict("enabled" => true, "interval_ms" => 10)) + +# Relative spread of the stage's Julia allocation total across the three orders +# that still counts as "the same cost". Measured: under 0.08% (see above). +const COST_TOLERANCE = 0.01 + +## Fixture +# A 64-bit LCG, so the fixture is the same bytes on every Julia version (the +# streams of Random's generators are not guaranteed to be). +mutable struct _LCG + s :: UInt64 +end +_next!(g::_LCG) = (g.s = g.s * 0x5851f42d4c957f2d + 0x14057b7ef767814f; g.s >> 33) + +const _NT = (UInt8('A'), UInt8('C'), UInt8('G'), UInt8('T')) +_randseq(g, n) = String([_NT[_next!(g) % 4 + 1] for _ in 1:n]) + +# A copy of `s` with about `per_1000` substitutions per thousand bases: close +# enough to its reference (~96% identity) to pass vsearch's --id 0.75. +function _mutate(g, s, per_1000) + b = Vector{UInt8}(s) + for i in eachindex(b) + _next!(g) % 1000 < per_1000 && (b[i] = _NT[_next!(g) % 4 + 1]) + end + String(b) +end + +function occupancy_fixture(dir; refs = 1000, asv = 300, otu = 150, len = 400) + g = _LCG(0x4d4d57) + mkpath(dir) + seqs = [_randseq(g, len) for _ in 1:refs] + open(joinpath(dir, "ref.fasta"), "w") do io + for (i, s) in enumerate(seqs) + println(io, ">ref$i;tax=k:K$(i % 7)") + println(io, s) + end + end + for (name, n) in (("asv", asv), ("otu", otu)) + open(joinpath(dir, "$name.fasta"), "w") do io + for i in 1:n + println(io, ">$name$i") + println(io, _mutate(g, seqs[_next!(g) % refs + 1], 40)) + end + end + end + dir +end + +## Running the stage in one order +function _search(dir, out, name) + TL.vsearch(joinpath(dir, "$name.fasta"), joinpath(dir, "ref.fasta"), joinpath(out, name); + optional_args = "--id 0.75 --query_cov 0.8 --threads 1", + vsearch_bin = something(Sys.which("vsearch"), "vsearch")) +end + +const ORDERS = ("asv_then_otu", "otu_then_asv", "concurrent") + +# The stage's work in `order`, under a fresh probe. Returns the probe's record. +function run_order(dir, out, order) + mkpath(out) + probe = MP.start(PROBE_ON) + MP.within(probe) do + if order == "asv_then_otu" + MP.task(() -> _search(dir, out, "asv"), "vsearch:asv") + MP.task(() -> _search(dir, out, "otu"), "vsearch:otu") + elseif order == "otu_then_asv" + MP.task(() -> _search(dir, out, "otu"), "vsearch:otu") + MP.task(() -> _search(dir, out, "asv"), "vsearch:asv") + else + # The shape of the full pipeline's vsearch stage. + a = Threads.@spawn MP.task(() -> _search(dir, out, "asv"), "vsearch:asv") + o = Threads.@spawn MP.task(() -> _search(dir, out, "otu"), "vsearch:otu") + fetch(a); fetch(o) + end + end + MP.finish!(probe) +end + +_task(rec, label) = only(t for t in rec["tasks"] if t["label"] == label) + +@testset "Occupancy probe (MMW-01)" begin + + @testset "off unless the run config enables it" begin + @test !MP.enabled(Dict()) + @test !MP.enabled(Dict("occupancy_probe" => Dict())) + @test !MP.enabled(Dict("occupancy_probe" => Dict("enabled" => "true"))) + @test !MP.enabled(Dict("occupancy_probe" => true)) + @test MP.enabled(PROBE_ON) + @test MP.start(Dict()) === nothing + @test MP.finish!(nothing) === nothing + @test MP.within(() -> 42, nothing) == 42 + @test MP.task(() -> 7, "outside a probe") == 7 + @test MP.interval_seconds(Dict()) == 0.010 + @test MP.interval_seconds(Dict("occupancy_probe" => Dict("interval_ms" => 25))) == 0.025 + @test MP.interval_seconds(Dict("occupancy_probe" => Dict("interval_ms" => 0))) == 0.010 + end + + @testset "flag off: the stage record is unchanged" begin + # The keys a stage record had before the probe existed. With the probe + # off, `finish!` gives `nothing` and the record must be exactly these. + before = ["name", "started", "finished", "status", "degraded", + "degraded_components", "runtimes", "tools", "databases", + "commands", "outputs"] + att = PV.Attestation() + PV.record_stage!(att, "vsearch", PV.CapturedEnvironment(); + occupancy = MP.finish!(MP.start(Dict()))) + @test collect(keys(only(att.doc["stages"]))) == before + + # Negative control: with the probe on, the key is there. + on = PV.Attestation() + PV.record_stage!(on, "vsearch", PV.CapturedEnvironment(); + occupancy = MP.finish!(MP.start(PROBE_ON))) + @test collect(keys(only(on.doc["stages"]))) == [before; "occupancy"] + end + + @testset "attestation round trip; older records still parse" begin + mktempdir() do tmp + att = PV.Attestation() + rec = MP.finish!(MP.start(PROBE_ON)) + PV.record_stage!(att, "vsearch", PV.CapturedEnvironment(); occupancy = rec) + PV.record_stage!(att, "swarm", PV.CapturedEnvironment()) # an unprobed stage + path = joinpath(tmp, "attestation.yml") + PV.write_attestation(att, path) + back = PV.read_attestation(path) + stages = Dict(s["name"] => s for s in back.doc["stages"]) + @test stages["vsearch"]["occupancy"]["probe"] == "MemoryProbe 1" + @test stages["vsearch"]["occupancy"]["interval_ms"] == 10 + @test !haskey(stages["swarm"], "occupancy") + + # A record written before the probe existed merges with a new one. + old = PV.Attestation() + PV.record_stage!(old, "cutadapt", PV.CapturedEnvironment()) + PV.write_attestation(old, path; merge_existing = false) + PV.write_attestation(att, path) + merged = Dict(s["name"] => s for s in PV.read_attestation(path).doc["stages"]) + @test !haskey(merged["cutadapt"], "occupancy") + @test haskey(merged["vsearch"], "occupancy") + end + end + + if !Sys.islinux() + @info "Occupancy probe: not Linux, so there is no /proc to sample; the child-memory tests are SKIPPED" + else + @testset "samples a child's resident memory" begin + probe = MP.start(PROBE_ON) + MP.within(probe) do + # bash holds a 64 MiB string for half a second. + MP.task("64MiB") do + TL._run_killable(`bash -c 'x=$(head -c 67108864 /dev/zero | tr "\0" a); sleep 0.5; : "${#x}"'`) + end + # Negative control: a child that allocates nothing. + MP.task("sleep") do + TL._run_killable(`sleep 0.5`) + end + end + rec = MP.finish!(probe) + big, small = _task(rec, "64MiB"), _task(rec, "sleep") + @test big["processes"] == 1 && big["samples"] > 0 + @test big["peak_rss_kib"] >= 64 * 1024 + @test big["hwm_kib"] >= big["peak_rss_kib"] + @test small["samples"] > 0 + @test 0 < small["peak_rss_kib"] < 16 * 1024 + # Serial: the stage peak is the larger task's, not the sum. + @test rec["child_peak_rss_kib"] < big["peak_rss_kib"] + small["peak_rss_kib"] + @test !haskey(rec, "error") + end + + @testset "through the pipeline's _attest, flag off and on" begin + SV = MetaManifold.Server + mktempdir() do tmp + cfg_path = joinpath(tmp, "run_config.yml") + for probed in (false, true) + cfg = Dict{String,Any}("vsearch" => Dict("enabled" => true)) + probed && (cfg["occupancy_probe"] = Dict("enabled" => true)) + YAML.write_file(cfg_path, cfg) + att = PV.Attestation(; config = cfg, config_sha256 = PV.file_sha256(cfg_path)) + SV._attest(att, PV.CapturedEnvironment(), "vsearch"; config_path = cfg_path) do + MetaManifold.Config._begin_section(cfg_path, "vsearch", joinpath(tmp, "vsearch.hash")) + MP.task(() -> TL._run_killable(`sleep 0.2`), "vsearch:asv") + nothing + end + stage = only(att.doc["stages"]) + @test stage["status"] == "ok" + @test haskey(stage, "occupancy") == probed + probed && @test only(stage["occupancy"]["tasks"])["label"] == "vsearch:asv" + + # A stage that throws is recorded as failed, with its + # occupancy when probed, and the error still propagates. + failed = PV.Attestation(; config = cfg, config_sha256 = PV.file_sha256(cfg_path)) + @test_throws ProcessFailedException SV._attest(failed, PV.CapturedEnvironment(), "swarm"; + config_path = cfg_path) do + TL._run_killable(`false`) + end + stage = only(failed.doc["stages"]) + @test stage["status"] == "failed" + @test haskey(stage, "occupancy") == probed + end + end + end + + @testset "a pipeline of commands counts each process" begin + probe = MP.start(PROBE_ON) + MP.within(() -> MP.task(() -> TL._run_killable(pipeline(`sleep 0.3`, `cat`)), "chain"), probe) + t = _task(MP.finish!(probe), "chain") + @test t["processes"] == 2 + @test t["samples"] > 0 + end + + @testset "a child that exits at once is counted, not sampled" begin + probe = MP.start(PROBE_ON) + MP.within(() -> MP.task(() -> TL._run_killable(`true`), "instant"), probe) + t = _task(MP.finish!(probe), "instant") + @test t["processes"] == 1 + @test t["samples"] == 0 || t["peak_rss_kib"] > 0 + end + + vsearch_bin = Sys.which("vsearch") + if isnothing(vsearch_bin) + @info "vsearch not on PATH - the two-thread disagreement test is SKIPPED" + else + @testset "two-thread disagreement: vsearch ASV and OTU in three orders" begin + mktempdir() do tmp + dir = occupancy_fixture(joinpath(tmp, "fixture")) + # Warm-up: compile every path once, so no order pays for it. + for order in ORDERS + run_order(dir, joinpath(tmp, "warmup", order), order) + end + GC.gc() + records = OrderedDict{String,Any}() + for order in ORDERS + GC.gc() + records[order] = run_order(dir, joinpath(tmp, order), order) + end + + # The same work: identical hits in every order. + for name in ("asv", "otu") + hits = [read(joinpath(tmp, o, name, "taxonomy.tsv"), String) for o in ORDERS] + @test !isempty(first(hits)) + @test allequal(hits) + end + + # Cost commutes (asserted). + cost = [records[o]["julia_allocated_bytes"] for o in ORDERS] + spread = (maximum(cost) - minimum(cost)) / maximum(cost) + @test spread <= COST_TOLERANCE + + # Peaks (recorded, not asserted). + for o in ORDERS + r = records[o] + a, u = _task(r, "vsearch:asv"), _task(r, "vsearch:otu") + @info "occupancy" order = o julia_allocated_bytes = r["julia_allocated_bytes"] julia_live_bytes_delta = r["julia_live_bytes_delta"] child_peak_rss_kib = r["child_peak_rss_kib"] asv_peak_rss_kib = a["peak_rss_kib"] asv_hwm_kib = a["hwm_kib"] otu_peak_rss_kib = u["peak_rss_kib"] otu_hwm_kib = u["hwm_kib"] samples = r["samples"] + end + conc = records["concurrent"] + serial_max = maximum(records[o]["child_peak_rss_kib"] for o in ORDERS[1:2]) + overlap = conc["child_peak_rss_kib"] > serial_max + @info "occupancy: concurrent stage peak vs the larger serial stage peak" concurrent = conc["child_peak_rss_kib"] serial = serial_max overlapped = overlap cost_spread = round(spread; sigdigits = 3) + overlap || @warn "occupancy: the concurrent searches did not overlap in memory on this runner, so this run cannot show the order dependence" + + path = get(ENV, "OCCUPANCY_RESULTS", "") + if !isempty(path) + io = IOBuffer() + run(pipeline(`$vsearch_bin --version`; stdout = io, stderr = io)) + version = first(split(String(take!(io)), '\n')) + YAML.write_file(path, OrderedDict{String,Any}( + "vsearch" => version, + "julia" => string(VERSION), "threads" => Threads.nthreads(), + "cpus" => Sys.CPU_THREADS, "cost_spread" => spread, + "orders" => records)) + end + end + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 4a8f47fa..a406baa1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -83,6 +83,12 @@ using MetaManifold.Categories, MetaManifold.CompositionLibrary include(joinpath(@__DIR__, "unit", f)) end + ## Occupancy probe and the two-thread disagreement test (MMW-01), with the + ## full suite. Its vsearch part skips, saying so, when vsearch is not on PATH. + if isempty(filter(a -> !startswith(a, "--"), ARGS)) + include("occupancy_disagreement_test.jl") + end + ## Integration tests (opt-in) if RUN_INTEGRATION using MetaManifold.DADA2, MetaManifold.OTUPipeline From 4dc19c05fd01963a21d53c39417896745c476cc1 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:46:01 +0100 Subject: [PATCH 02/45] docs(occupancy): dated reading of the vsearch stage, with a gate (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MMW-01, part 2 of 2. Stacked on hyperpolymath/MetaManifold-WebUI#106 (the probe and the test). This part is fork-side, about the estate's proof siblings, and is not meant for the upstream PR. ## Summary **`docs/occupancy-2026-10.adoc`** is the dated table (2026-10-14). For each order it gives: - each search's peak; - the measured stage peak; - the Session IR counted bytes; - the HWM typed peak; - the Julia allocation. It also has a status ledger, the limits, and what the next proofs would change. | order | stage peak, measured | counted, Session IR | typed peak, HWM | |---|---|---|---| | ASV then OTU | 9204 | 9204 | 9204 | | OTU then ASV | 9192 | 9192 | 9192 | | concurrent | 18380 | 18380 | 18380 | (KiB.) **The pieces:** - **`docs/occupancy/vsearch-*.sir`:** the stage as occupancy-types Session IR, one buffer per search, in each order. - **`docs/occupancy/TypedPeak.agda`:** the typed peak, computed by `refl` with occupancy-types' `Timeline` over resource-algebra's HWM grade. - A search alone exits holding nothing (drift 0). - The first concurrent search is still resident when the second starts (drift = peak). - `docs/occupancy/reject/` holds two claims that must be refused: *concurrent = serial*, and *serial = sum*, which reads occupancy as cost (R-6). - **`scripts/fetch-siblings.sh`:** pins occupancy-types `af95728` and resource-algebra `cf9235c` (the CONTEXT pins) into `_build/dependencies`. Their definitions are consumed, never copied. occupancy-types type-checks against both resource-algebra `cf9235c` and its own pin `b2c9f1f`. - **`scripts/check-occupancy.sh`:** the gate. It runs, in order: - Session IR `check` and `run --trace`; - `TypedPeak.agda`; - the two rejections, each required to fail at its own declaration; - counted bytes against the typed peak, order by order. - **`.github/workflows/occupancy.yml`:** runs the gate when these files change, on any base branch, so this stacked PR is checked too. It uses the digest-pinned Debian trixie image with Agda 2.6.4.3 that upstream's proof job uses, plus `setup-julia` 1.12.5. - **`docs/occupancy/results-2026-10-14.yml`:** the raw probe records behind the table. - **`.gitignore`:** unignores exactly these docs (`docs/*` is otherwise kept local), and ignores `_build/` and `*.agdai`. ## Status rows | ID | Claim | Status | |---|---|---| | MMW-01.1 | flag off ⇒ record unchanged; on ⇒ `occupancy`; old records parse | TESTED (#106's test file) | | MMW-01.2 | cost commutes for this workload | TESTED (asserted ≤ 1%) | | MMW-01.3 | stage peak order-dependent, task peaks not (A-3) for this workload | TESTED (recorded) | | MMW-01.4 | counted bytes = typed peak for the three programs | TESTED (gate) | | bridge | counted bytes ≤ typed peak in general | OPEN (unchanged: three instances, not a proof) | ## Verification - `bash scripts/check-occupancy.sh` exits 0 locally. It ran with Agda from the PyPI 2.6.4.3 wheel (which reports itself as 2.6.4.1) and Julia 1.12.5. CI runs the Debian 2.6.4.3 package. - Negative controls, each failing the gate: - one byte changed in a Session IR text (`18381 != 18380`); - a true claim planted as a rejection ("invalid claim accepted"); - a rejection failing on a scope error ("failed for the wrong reason"). - `zizmor` finds nothing in the workflow. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_015pMjy981ie74uCttyYDanC --- _Generated by [Claude Code](https://claude.ai/code/session_015pMjy981ie74uCttyYDanC)_ Co-authored-by: Claude Opus 5.5 --- .github/workflows/occupancy.yml | 64 +++++++++ .gitignore | 7 + docs/occupancy-2026-10.adoc | 125 ++++++++++++++++++ docs/occupancy/TypedPeak.agda | 45 +++++++ docs/occupancy/reject/ConcurrentIsSerial.agda | 14 ++ docs/occupancy/reject/SerialIsCost.agda | 14 ++ docs/occupancy/results-2026-10-14.yml | 63 +++++++++ docs/occupancy/vsearch-asv-then-otu.sir | 13 ++ docs/occupancy/vsearch-concurrent.sir | 12 ++ docs/occupancy/vsearch-otu-then-asv.sir | 9 ++ scripts/check-occupancy.sh | 69 ++++++++++ scripts/fetch-siblings.sh | 29 ++++ 12 files changed, 464 insertions(+) create mode 100644 .github/workflows/occupancy.yml create mode 100644 docs/occupancy-2026-10.adoc create mode 100644 docs/occupancy/TypedPeak.agda create mode 100644 docs/occupancy/reject/ConcurrentIsSerial.agda create mode 100644 docs/occupancy/reject/SerialIsCost.agda create mode 100644 docs/occupancy/results-2026-10-14.yml create mode 100644 docs/occupancy/vsearch-asv-then-otu.sir create mode 100644 docs/occupancy/vsearch-concurrent.sir create mode 100644 docs/occupancy/vsearch-otu-then-asv.sir create mode 100755 scripts/check-occupancy.sh create mode 100755 scripts/fetch-siblings.sh diff --git a/.github/workflows/occupancy.yml b/.github/workflows/occupancy.yml new file mode 100644 index 00000000..a1c7136d --- /dev/null +++ b/.github/workflows/occupancy.yml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# The occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): +# Session IR's counted bytes against the typed peak computed in Agda from the +# pinned proof siblings. Kept apart from ci.yml because it is about the docs and +# their pinned siblings, not the application; it runs only when they change. +name: Occupancy reading + +on: + push: + branches: [main] + paths: + - docs/occupancy/** + - docs/occupancy-2026-10.adoc + - scripts/check-occupancy.sh + - scripts/fetch-siblings.sh + - .github/workflows/occupancy.yml + # Any base branch, so a PR stacked on another one is checked too. + pull_request: + paths: + - docs/occupancy/** + - docs/occupancy-2026-10.adoc + - scripts/check-occupancy.sh + - scripts/fetch-siblings.sh + - .github/workflows/occupancy.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'push' }} + +jobs: + typed-peak: + name: Occupancy typed peak + runs-on: ubuntu-24.04 + # The Agda 2.6.4.3 Debian trixie packages, in a digest-pinned image, as the + # upstream proof job uses. The siblings need no standard library. + container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb + timeout-minutes: 20 + env: + DEBIAN_FRONTEND: noninteractive + steps: + # git before checkout, so the checkout is a repository and the siblings can be fetched. + - name: Install git and Agda + run: | + apt-get update -qq + apt-get install -y --no-install-recommends ca-certificates git agda-bin=2.6.4.3-1+b2 + agda --version | grep -qx 'Agda version 2.6.4.3' + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false # the siblings are public; nothing here pushes + + # The Session IR stepper is Julia (standard library only). + - name: Set up Julia + uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 + with: + version: "1.12.5" + + - name: Counted bytes against typed peak + run: bash scripts/check-occupancy.sh diff --git a/.gitignore b/.gitignore index a6ed2b7d..1436ebab 100644 --- a/.gitignore +++ b/.gitignore @@ -250,6 +250,9 @@ vignettes/*.pdf ## Documentation kept local except published release notes docs/* !docs/release-notes/ +# and the dated occupancy reading (MMW-01), whose table the gate checks +!docs/occupancy-2026-10.adoc +!docs/occupancy/ # translation temp files @@ -332,3 +335,7 @@ LocalPreferences.toml # Development server log (dev/dev.sh) dev/server.log + +# Proof siblings fetched by scripts/fetch-siblings.sh, and Agda interface files +/_build/ +*.agdai diff --git a/docs/occupancy-2026-10.adoc b/docs/occupancy-2026-10.adoc new file mode 100644 index 00000000..b7323e45 --- /dev/null +++ b/docs/occupancy-2026-10.adoc @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) += Occupancy of the vsearch stage +:revdate: 2026-10-14 + +The vsearch stage runs two independent searches, ASV and OTU. The full +pipeline spawns them together; a single-stage re-run runs them one after the +other. The high-water-mark (HWM) algebra of occupancy-types predicts that the +stage's cost is the same in every order but its memory peak is not: serially +the peak is the larger search's, concurrently it can be their sum +(choreographic-types A-2 and A-3). This page records one measurement of that, +on 2026-10-14, and checks it against the typed reading. + +== What was measured + +`MemoryProbe` (`src/core/memory_probe.jl`), off unless a pipeline.yml sets +`occupancy_probe.enabled: true`, records for each attested stage: + +* `julia_allocated_bytes`: the `Base.gc_num()` delta over the stage (cost); +* per task (one spawned search): the largest sampled VmRSS of the child + process it started (`peak_rss_kib`), and the kernel's own VmHWM for it + (`hwm_kib`); +* `child_peak_rss_kib`: the largest sum of the stage's children's VmRSS at + one sampling instant (the stage's occupancy). + +The run: `test/occupancy_disagreement_test.jl`, which runs the same two searches +in three orders under the probe, on a fixture it generates: 1000 reference +sequences of 400 bp, 300 ASV and 150 OTU queries. Linux 6.18 with 4 CPUs, +Julia 1.12.5 with 2 threads, vsearch 2.30.5 with `--threads 1`. Sampling +interval 10 ms. + +== Results + +[cols="2,1,1,1,1,1,1",options="header"] +|=== +| Order | ASV peak (KiB) | OTU peak (KiB) | Stage peak, measured (KiB) | Counted bytes, Session IR | Typed peak, HWM | Julia allocation (bytes) +| ASV then OTU | 9204 | 9180 | 9204 | 9204 | 9204 | 978432 +| OTU then ASV | 9152 | 9192 | 9192 | 9192 | 9192 | 978720 +| Concurrent | 9176 | 9204 | 18380 | 18380 | 18380 | 978816 +|=== + +* *Cost commutes.* The allocation totals agree to 0.04%. The test asserts + agreement within 1%; a planted extra search in one order breaks it (spread + 25%). +* *Each search's peak is the same in every order*, to within 0.6% + (9152-9204 KiB). Each search's VmHWM equals its sampled peak, so the + sampling caught the peaks. +* *The stage peak is not the same.* Serially it is the larger search's + (9192-9204 KiB); concurrently it is 18380 KiB, twice that. This is the A-3 + order dependence, and nothing in the stage's outputs shows it: the hits are + byte-identical in all three orders. The prediction would have been falsified + had the concurrent peak matched the serial one; it did not. +* *Counted bytes equal the typed peak*, in all three orders. The counted bytes + come from occupancy-types' Session IR stepper running + `docs/occupancy/vsearch-*.sir`, one buffer per search. The typed peak is + computed, by `refl`, from occupancy-types' `Timeline` over resource-algebra's + HWM grade (`docs/occupancy/TypedPeak.agda`). A search alone exits holding + nothing (drift 0); in the concurrent order the first is still resident + when the second starts (drift = peak). The sequence law + `max (peak a) (drift a + peak b)` then gives the larger peak serially and the + sum concurrently. +* *The measured stage peak also equals the typed peak* here. For the concurrent + order the typed peak is the interleaving in which both are resident at once; + a measurement could come in lower if the two searches' peaks did not + coincide. Here they did: each search's memory plateaus once its database is + loaded. + +== Status + +[cols="1,3,1,2",options="header"] +|=== +| ID | Claim | Status | Artefact +| MMW-01.1 | With the flag off, a stage record has exactly its earlier keys; with it on, it gains `occupancy`; old records still parse and merge | TESTED | `test/occupancy_disagreement_test.jl` ("flag off", "round trip", "through the pipeline's _attest") +| MMW-01.2 | For this workload, the vsearch stage's cost (Julia allocation) is the same in all three orders | TESTED | same file, "two-thread disagreement" (asserted, within 1%) +| MMW-01.3 | For this workload, the stage peak depends on the order (A-3), and each search's own peak does not | TESTED | same file, recorded in the test log; table above +| MMW-01.4 | For the three Session IR texts, the stepper's counted bytes equal the HWM typed peak | TESTED | `scripts/check-occupancy.sh` (stepper output against `TypedPeak.agda`; two rejections in `docs/occupancy/reject`) +| bridge | Counted bytes are bounded by the typed peak in general (occupancy-types' v0 → typed bridge) | OPEN | unchanged; this page gives three instances, not a proof +|=== + +== Limits + +* *Sampled.* A per-task peak is the largest VmRSS seen at 10 ms intervals. VmHWM + is exact up to the last sample, but growth after it is not seen. A child that + lives for less than one interval may not be sampled at all (`samples: 0`). +* *Child memory only.* The children are counted; the Julia process is not. Its + `julia_live_bytes_delta` is recorded, but without a forced collection it + reflects when the GC last ran, not what the stage kept. +* *Allocation is process-wide.* While the study loop runs several runs' + pipelines at once, a stage's `julia_allocated_bytes` includes the others' + allocations too. The child figures are per stage, since each stage samples + only the children it started. +* *Linux only*, since it reads /proc. For a stage offloaded with `remote:`, + the local child is an ssh client, so its memory says nothing about the work. +* *One thread for vsearch.* With several threads, vsearch writes its hits in + whatever order the threads finish. `_sort_hits_by_query!` then does a + different amount of work, because its sort key splits each line on every + comparison. That moved the stage's allocation total by up to 12% between + repeats of the same order. So in production the cost is order-independent + only up to that noise; precomputing the sort key would remove it. This PR + does not change it. +* *One host, one day.* CI runs the same test on every push and logs its own + numbers; they are not compared with these. + +== Reproducing + +---- +julia --project=. -t 2 test/occupancy_disagreement_test.jl # set OCCUPANCY_RESULTS= for YAML +bash scripts/check-occupancy.sh # needs julia and agda 2.6.4.x +---- + +`scripts/fetch-siblings.sh` fetches the siblings at their pins into +`_build/dependencies`: occupancy-types `af95728` and resource-algebra +`cf9235c`. + +== As the proofs progress + +* *Per-buffer (n-ary) occupancy* in occupancy-types: each search is already one + buffer in the Session IR texts, so the typed peak could then be read per + search. +* *Signed drift*: frees could be graded as negative drift, so the Julia side's + collections (the pipeline's `GC.gc(false)` hints) become modelled + reclamation rather than unmodelled noise. +* *choreographic-types' operational `step` as a function*: the three orders + become three traces of one typed protocol. The frontier certificate its + two-thread application note describes could then be displayed in the WebUI. diff --git a/docs/occupancy/TypedPeak.agda b/docs/occupancy/TypedPeak.agda new file mode 100644 index 00000000..6899efb2 --- /dev/null +++ b/docs/occupancy/TypedPeak.agda @@ -0,0 +1,45 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- The vsearch stage's typed peak, computed (TESTED, by refl) with +-- occupancy-types' Timeline over resource-algebra's HWM grade, both consumed +-- by pin through scripts/fetch-siblings.sh: +-- occupancy-types af957280075699c659de0abd9a3b0defade77309 +-- resource-algebra cf9235c532712a783a404c359a1055f40134794a +-- Nothing here re-declares their definitions. +-- +-- Each search is one interaction graded by its measured peak, in KiB, from the +-- 2026-10-14 run in docs/occupancy-2026-10.adoc. A search's process exits +-- holding nothing, so run alone its drift is 0. Run concurrently, the first is +-- still resident when the second starts, so that is graded with drift = peak. +-- That is the interleaving the measurement observed (both resident at once). +-- The algebra's sequence law, max (peak a) (drift a + peak b), then gives: +-- serial = the larger peak; concurrent = the sum. +module TypedPeak where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM + +import OccupancyTypes.Timeline +module TH = OccupancyTypes.Timeline HWM + +-- A search whose process has exited before the next one starts. +exits : Nat → Grade +exits p = hwm p 0 z≤n + +-- A search still resident when the next one starts. +resident : Nat → Grade +resident p = hwm p p (≤-refl p) + +-- ASV 9204 then OTU 9180. +asv-then-otu : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 9204 +asv-then-otu = refl + +-- OTU 9192 then ASV 9152. +otu-then-asv : peak (TH.grade (exits 9192 TH.∷ exits 9152 TH.∷ TH.[])) ≡ 9192 +otu-then-asv = refl + +-- ASV 9176 and OTU 9204 together. +concurrent : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 18380 +concurrent = refl diff --git a/docs/occupancy/reject/ConcurrentIsSerial.agda b/docs/occupancy/reject/ConcurrentIsSerial.agda new file mode 100644 index 00000000..592de6ee --- /dev/null +++ b/docs/occupancy/reject/ConcurrentIsSerial.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: the claim that running the two searches concurrently leaves +-- the peak where a serial run puts it (the larger of the two). +module ConcurrentIsSerial where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 9204 +wrong = refl diff --git a/docs/occupancy/reject/SerialIsCost.agda b/docs/occupancy/reject/SerialIsCost.agda new file mode 100644 index 00000000..dbb6f4ba --- /dev/null +++ b/docs/occupancy/reject/SerialIsCost.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: reading a serial run's peak as the sum of the two peaks, +-- i.e. occupancy as cost (occupancy-types R-6). +module SerialIsCost where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 18384 +wrong = refl diff --git a/docs/occupancy/results-2026-10-14.yml b/docs/occupancy/results-2026-10-14.yml new file mode 100644 index 00000000..394236d6 --- /dev/null +++ b/docs/occupancy/results-2026-10-14.yml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# The raw MemoryProbe records behind docs/occupancy-2026-10.adoc, written by +# test/occupancy_disagreement_test.jl with OCCUPANCY_RESULTS set (2026-10-14). +vsearch: "vsearch v2.30.5_linux_x86_64, 15.7GB RAM, 4 cores" +julia: "1.12.5" +threads: 2 +cpus: 4 +cost_spread: 0.00039231071008238524 +orders: + asv_then_otu: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 56 + julia_allocated_bytes: 978432 + julia_live_bytes_delta: 978544 + child_peak_rss_kib: 9204 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9204 + hwm_kib: 9204 + - label: "vsearch:otu" + processes: 1 + samples: 20 + peak_rss_kib: 9180 + hwm_kib: 9180 + otu_then_asv: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 57 + julia_allocated_bytes: 978720 + julia_live_bytes_delta: 978832 + child_peak_rss_kib: 9192 + tasks: + - label: "vsearch:otu" + processes: 1 + samples: 22 + peak_rss_kib: 9192 + hwm_kib: 9192 + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9152 + hwm_kib: 9152 + concurrent: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 33 + julia_allocated_bytes: 978816 + julia_live_bytes_delta: 978928 + child_peak_rss_kib: 18380 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 30 + peak_rss_kib: 9176 + hwm_kib: 9176 + - label: "vsearch:otu" + processes: 1 + samples: 24 + peak_rss_kib: 9204 + hwm_kib: 9204 diff --git a/docs/occupancy/vsearch-asv-then-otu.sir b/docs/occupancy/vsearch-asv-then-otu.sir new file mode 100644 index 00000000..e462fb5e --- /dev/null +++ b/docs/occupancy/vsearch-asv-then-otu.sir @@ -0,0 +1,13 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-asv-then-otu.sir - the vsearch stage with the two searches run one + after the other, ASV first: the single-stage re-run's order. Each search is + one buffer, sized by its measured peak in KiB (docs/occupancy-2026-10.adoc, + 2026-10-14 run); each process exits holding nothing, so its buffer is dropped + before the next search starts. + + Expected: steps = 0 (no communication), peak_live = max 9204 9180 = 9204. *) +let asv = alloc 9204 in +let _ = drop asv in +let otu = alloc 9180 in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-concurrent.sir b/docs/occupancy/vsearch-concurrent.sir new file mode 100644 index 00000000..93be40bf --- /dev/null +++ b/docs/occupancy/vsearch-concurrent.sir @@ -0,0 +1,12 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-concurrent.sir - the two searches spawned together, as the full + pipeline runs them. Session IR has no parallel composition, so this is the + interleaving the measurement observed: both buffers live at once, then both + dropped. + + Expected: steps = 0, peak_live = 9176 + 9204 = 18380. *) +let asv = alloc 9176 in +let otu = alloc 9204 in +let _ = drop asv in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-otu-then-asv.sir b/docs/occupancy/vsearch-otu-then-asv.sir new file mode 100644 index 00000000..ba575591 --- /dev/null +++ b/docs/occupancy/vsearch-otu-then-asv.sir @@ -0,0 +1,9 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-otu-then-asv.sir - the same two searches, OTU first. + + Expected: steps = 0, peak_live = max 9192 9152 = 9192. *) +let otu = alloc 9192 in +let _ = drop otu in +let asv = alloc 9152 in +let _ = drop asv in +unit diff --git a/scripts/check-occupancy.sh b/scripts/check-occupancy.sh new file mode 100755 index 00000000..f9bcba36 --- /dev/null +++ b/scripts/check-occupancy.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Gate for the occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): +# 1. each Session IR text checks, and occupancy-types' stepper counts its bytes; +# 2. TypedPeak.agda type-checks against the pinned siblings, and each module in +# docs/occupancy/reject is refused at its own declaration; +# 3. the counted bytes equal the typed peak, order by order. +# Needs julia and agda (2.6.4.x) on PATH. Exit 0 only if all three hold. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +export LC_ALL="${LC_ALL:-C.UTF-8}" + +bash scripts/fetch-siblings.sh +deps=_build/dependencies +sir="$deps/occupancy-types/src/julia" +agda --version | head -1 +flags=(--safe --without-K --no-libraries --ignore-interfaces --double-check + -i docs/occupancy -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") +logs=_build/check-logs +mkdir -p "$logs" +failures=0 + +# Order -> Session IR file and TypedPeak lemma. +orders=(asv-then-otu otu-then-asv concurrent) + +echo "== Session IR (occupancy-types v0 stepper)" +declare -A counted +for o in "${orders[@]}"; do + file="docs/occupancy/vsearch-$o.sir" + julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" check "$file" + out="$(julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" run "$file" --trace)" + echo "$out" + counted[$o]="$(sed -n 's/.*peak_live=\([0-9]*\).*/\1/p' <<<"$out" | head -1)" +done + +echo "== Typed peak (Agda, HWM Timeline)" +agda "${flags[@]}" docs/occupancy/TypedPeak.agda >"$logs/TypedPeak.log" 2>&1 || + { cat "$logs/TypedPeak.log"; echo "ERROR: TypedPeak.agda does not type-check" >&2; exit 1; } +echo "PASS: TypedPeak.agda" +for file in docs/occupancy/reject/*.agda; do + name="$(basename -- "$file" .agda)" + log="$logs/$name.log" + if agda "${flags[@]}" -i docs/occupancy/reject "$file" >"$log" 2>&1; then + cat "$log"; echo "ERROR: invalid claim accepted: $name" >&2; failures=$((failures + 1)); continue + fi + if ! grep -Eq "/$name\.agda:[0-9]+,[0-9]+" "$log" || ! grep -q 'when checking that' "$log" || + grep -Eq 'Not in scope|Failed to find|Unsolved|Parse error' "$log"; then + cat "$log"; echo "ERROR: $name failed for the wrong reason" >&2; failures=$((failures + 1)); continue + fi + echo "PASS: rejected $name ($(grep -m1 -E '[0-9]+ != [0-9]+' "$log"))" +done + +echo "== Counted bytes against typed peak (KiB)" +printf '%-14s %10s %10s\n' order counted typed +for o in "${orders[@]}"; do + typed="$(grep -A1 -E "^$o :" docs/occupancy/TypedPeak.agda | sed -n 's/.*≡ \([0-9]*\)$/\1/p' | head -1)" + printf '%-14s %10s %10s\n' "$o" "${counted[$o]}" "$typed" + if [ -z "${counted[$o]}" ] || [ -z "$typed" ] || [ "${counted[$o]}" != "$typed" ]; then + echo "ERROR: $o: counted ${counted[$o]:-?} != typed ${typed:-?}" >&2; failures=$((failures + 1)) + fi +done + +if (( failures > 0 )); then + echo "check-occupancy: $failures failure(s)" >&2 + exit 1 +fi +echo "check-occupancy: OK" diff --git a/scripts/fetch-siblings.sh b/scripts/fetch-siblings.sh new file mode 100755 index 00000000..31a98bc1 --- /dev/null +++ b/scripts/fetch-siblings.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fetch the proof siblings the occupancy reading (docs/occupancy-2026-10.adoc) +# consumes, each at its pinned commit, into _build/dependencies. Their +# definitions are used, never copied. Re-pin one sibling per commit. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +# occupancy-types: the HWM Timeline (Agda) and the SessionIR stepper (Julia). +OCCUPANCY_TYPES_COMMIT="${OCCUPANCY_TYPES_COMMIT:-af957280075699c659de0abd9a3b0defade77309}" +# resource-algebra: the HWM grade algebra occupancy-types builds on. +RESOURCE_ALGEBRA_COMMIT="${RESOURCE_ALGEBRA_COMMIT:-cf9235c532712a783a404c359a1055f40134794a}" + +fetch() { + local name="$1" commit="$2" dir="_build/dependencies/$1" + if [ ! -d "$dir/.git" ]; then + git init --quiet "$dir" + git -C "$dir" remote add origin "https://github.com/hyperpolymath/$name.git" + fi + git -C "$dir" fetch --quiet --depth 1 origin "$commit" + git -C "$dir" checkout --quiet --detach "$commit" + echo "$name at $(git -C "$dir" rev-parse --short HEAD)" +} + +mkdir -p _build/dependencies +fetch occupancy-types "$OCCUPANCY_TYPES_COMMIT" +fetch resource-algebra "$RESOURCE_ALGEBRA_COMMIT" From 91f3bb83fd8259a85c9c1f5a10f0e78c9f585800 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:56:06 +0100 Subject: [PATCH 03/45] ci(occupancy): install from pinned tarballs, and run the disagreement test here (#108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to hyperpolymath/MetaManifold-WebUI#106 and hyperpolymath/MetaManifold-WebUI#107. Both were merged before their checks ran, so this is the part that never ran. ## Summary **The problem.** The `Occupancy reading` workflow ends in `startup_failure`: on #107's PR run, and again on `main` at `4dc19c0`. This account's Actions policy allows only GitHub's own actions and hyperpolymath's own, pinned to a full commit SHA, and `julia-actions/setup-julia` is neither. The same policy is why `ci.yml` has not started on this fork since at least 2026-10-02: it uses `julia-actions/*`, `oven-sh/setup-bun` and `codecov/codecov-action`. So the three-order test that #106 added has never run here. **The fix.** The workflow now uses no third-party action. It installs: - **Julia 1.12.5** from julialang's tarball, checked against julialang's published SHA256 (`41b84d72…8bae7`, as docker-library/julia `a39df01` records it); - **vsearch** from its `config/defaults/tool_versions.yml` pin, checked against the pinned SHA256, exactly as `ci.yml` does. It has two jobs: - **Occupancy disagreement test:** Julia, R from apt (RCall builds against it), `Pkg.instantiate()`, the pinned vsearch, then `julia --project=. -t 2 test/occupancy_disagreement_test.jl`; - **Occupancy typed peak:** `scripts/check-occupancy.sh` in the digest-pinned Debian trixie image with Agda 2.6.4.3, as before. The path filters now also cover the probe's sources and the test. ## Verification - `zizmor` finds nothing. - The only `uses:` is `actions/checkout`, SHA-pinned. - The `eval` of the pin-reading Julia snippet was rehearsed locally: `2.30.5`, the pinned URL and the SHA256. - This PR's own run of the workflow is the real check. The fork's `ci.yml` will keep failing at startup whatever this PR does. The fix there is an account or repo setting (Settings → Actions → General) that allows those actions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_015pMjy981ie74uCttyYDanC --- _Generated by [Claude Code](https://claude.ai/code/session_015pMjy981ie74uCttyYDanC)_ Co-authored-by: Claude Opus 5.5 --- .github/workflows/occupancy.yml | 99 ++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 9 deletions(-) diff --git a/.github/workflows/occupancy.yml b/.github/workflows/occupancy.yml index a1c7136d..92433886 100644 --- a/.github/workflows/occupancy.yml +++ b/.github/workflows/occupancy.yml @@ -2,15 +2,26 @@ # SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) # # The occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): -# Session IR's counted bytes against the typed peak computed in Agda from the -# pinned proof siblings. Kept apart from ci.yml because it is about the docs and -# their pinned siblings, not the application; it runs only when they change. +# * the two-thread disagreement test (test/occupancy_disagreement_test.jl) on +# the pinned vsearch, and +# * Session IR's counted bytes against the typed peak computed in Agda from +# the pinned proof siblings (scripts/check-occupancy.sh). +# +# This account allows only GitHub's own actions and its own, pinned by SHA, so +# ci.yml (julia-actions, setup-bun, codecov) cannot start on this fork. That is +# why everything here installs from pinned URLs checked against pinned SHA256s, +# and why the disagreement test runs here as well as in ci.yml's full suite. name: Occupancy reading on: push: branches: [main] paths: + - src/core/memory_probe.jl + - src/core/provenance.jl + - src/pipeline/tools.jl + - src/server/routes/pipeline.jl + - test/occupancy_disagreement_test.jl - docs/occupancy/** - docs/occupancy-2026-10.adoc - scripts/check-occupancy.sh @@ -19,6 +30,11 @@ on: # Any base branch, so a PR stacked on another one is checked too. pull_request: paths: + - src/core/memory_probe.jl + - src/core/provenance.jl + - src/pipeline/tools.jl + - src/server/routes/pipeline.jl + - test/occupancy_disagreement_test.jl - docs/occupancy/** - docs/occupancy-2026-10.adoc - scripts/check-occupancy.sh @@ -32,7 +48,65 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name != 'push' }} +env: + # Julia 1.12.5, the version Manifest.toml was resolved with. The checksum is + # julialang's own (bin/checksums/julia-1.12.5.sha256), as the official Docker + # image (docker-library/julia a39df01) records it. + JULIA_URL: https://julialang-s3.julialang.org/bin/linux/x64/1.12/julia-1.12.5-linux-x86_64.tar.gz + JULIA_SHA256: 41b84d727e4e96fbf3ed9e92fa195d773d247b9097f73fad688f8b699758bae7 + jobs: + disagreement: + name: Occupancy disagreement test + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + # As in ci.yml: the root .Rprofile would otherwise bootstrap renv. + RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - name: Install Julia + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$JULIA_URL" julia.tar.gz + echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - + sudo mkdir -p /opt/julia + sudo tar xzf julia.tar.gz -C /opt/julia --strip-components=1 + rm julia.tar.gz + echo /opt/julia/bin >> "$GITHUB_PATH" + + # RCall builds against R. The occupancy test uses no R package. + - name: Install R + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends r-base-core + + - name: Instantiate the project + run: julia --project=. -e 'using Pkg; Pkg.instantiate()' + + # The pinned vsearch (config/defaults/tool_versions.yml), refused unless it + # hashes to the pinned SHA256, as ci.yml installs it. + - name: Install vsearch + run: | + eval "$(julia --project=. -e ' + using YAML + v = YAML.load_file("config/defaults/tool_versions.yml")["tools"]["vsearch"] + a = v["archives"]["linux-x86_64"] + println("VSEARCH_VERSION=", v["version"]); println("VSEARCH_URL=", a["url"]); println("VSEARCH_SHA256=", a["sha256"])')" + source scripts/ci/fetch_pinned.sh + fetch_pinned "$VSEARCH_URL" vsearch.tar.gz + echo "$VSEARCH_SHA256 vsearch.tar.gz" | sha256sum -c - + tar xzf vsearch.tar.gz --warning=no-unknown-keyword + sudo mv "vsearch-$VSEARCH_VERSION-linux-x86_64/bin/vsearch" /usr/local/bin/vsearch + rm -rf vsearch.tar.gz "vsearch-$VSEARCH_VERSION-linux-x86_64" + vsearch --version + + - name: Three orders of the vsearch stage + run: julia --project=. -t 2 test/occupancy_disagreement_test.jl + typed-peak: name: Occupancy typed peak runs-on: ubuntu-24.04 @@ -44,10 +118,10 @@ jobs: DEBIAN_FRONTEND: noninteractive steps: # git before checkout, so the checkout is a repository and the siblings can be fetched. - - name: Install git and Agda + - name: Install git, curl and Agda run: | apt-get update -qq - apt-get install -y --no-install-recommends ca-certificates git agda-bin=2.6.4.3-1+b2 + apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 agda --version | grep -qx 'Agda version 2.6.4.3' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -55,10 +129,17 @@ jobs: persist-credentials: false # the siblings are public; nothing here pushes # The Session IR stepper is Julia (standard library only). - - name: Set up Julia - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 - with: - version: "1.12.5" + - name: Install Julia + shell: bash + run: | + source scripts/ci/fetch_pinned.sh + fetch_pinned "$JULIA_URL" julia.tar.gz + echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - + mkdir -p /opt/julia + tar xzf julia.tar.gz -C /opt/julia --strip-components=1 + rm julia.tar.gz + echo /opt/julia/bin >> "$GITHUB_PATH" - name: Counted bytes against typed peak + shell: bash run: bash scripts/check-occupancy.sh From 44a2273d4ead308a1811cbd5f54d5ae6d0bb57be Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:34:25 +0100 Subject: [PATCH 04/45] chore(sync): merge JoshuaJewell/MetaManifold-WebUI main (bcf52af) into the fork (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The fork's `main` is upstream `257bcde` plus the three MMW-01 PRs (#106, #107, #108). Upstream has 25 commits since, up to `bcf52af`. Among them: - the PERMANOVA per-term test (JoshuaJewell/MetaManifold-WebUI#35); - the Agda proofs job; - the stable check name; - the config write gates; - clr_welch. This merges them in. It does not rebase, so `main` keeps its history and nothing is force-pushed. It is the base for the upstream PR of the probe (branch `feat/occupancy-probe`), whose files are byte-identical to this branch's. **Conflicts.** There were two. Both were additions on each side, so both sides are kept, upstream first: - **`.gitignore`:** upstream's `proofs/agda/_build/`, then the fork's `/_build/` and `*.agdai` for the fetched proof siblings. - **`config/defaults/pipeline.yml`:** upstream's `differential.replacement_delta`, then the fork's commented-out `occupancy_probe` block at the end of the file. ## Verification The unit suite ran on this tree with Julia 1.12.5, vsearch 2.30.5 and `-t 2`. - **Totals:** 2408 passed, 0 failed, 20 broken (tests the suite marks as known-broken), 1 error. - **The error is environmental.** `live probes of what is actually installed` (test/unit/test_provenance.jl:478) reports `package 'dada2' is not installed in the R library in use`. That sandbox has no dada2; CI restores it from `renv.lock`. - **New upstream tests:** "Config write gate" passes 54/54 and "Pipeline tables cannot be deleted" 24/24. - **Occupancy probe:** 46/46, the test from #106. **Expected CI on this PR:** - **`CI`:** `startup_failure`. This branch still carries upstream's `ci.yml`, whose julia-actions, setup-bun and codecov steps this account's Actions policy refuses. The stacked PR `ci/no-third-party-actions` fixes that and is where `ci.yml` is actually tested, on this merge plus its own commit. - **`Occupancy reading`:** does not run. Its paths are untouched by the merge. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_015pMjy981ie74uCttyYDanC --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Joshua Jewell Co-authored-by: x Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 95 ++- .gitignore | 2 + CONTRIBUTING.md | 2 +- config/defaults/pipeline.yml | 16 +- frontend/src/api/types.ts | 56 +- frontend/src/components/AnalysisWorkspace.tsx | 6 +- frontend/src/components/CompositionPanel.tsx | 10 + frontend/src/components/DifferentialPanel.tsx | 73 ++- frontend/src/components/DiversityPanel.tsx | 59 +- frontend/src/components/PipelineStages.tsx | 6 +- .../src/components/TaxaCompositionChart.tsx | 50 +- frontend/src/components/VennPanel.tsx | 49 +- .../src/components/annotationShared.test.ts | 38 ++ frontend/src/components/annotationShared.ts | 61 +- .../src/components/differentialTable.test.ts | 26 +- frontend/src/components/differentialTable.ts | 21 +- frontend/src/components/sharedRanks.test.ts | 36 ++ frontend/src/components/sharedRanks.ts | 17 + frontend/src/hooks/useApi.test.ts | 16 + frontend/src/hooks/useApi.ts | 31 +- proofs/agda/MetaManifold/All.agda | 14 + .../agda/MetaManifold/BenjaminiHochberg.agda | 582 ++++++++++++++++++ proofs/agda/MetaManifold/Prelude.agda | 83 +++ proofs/agda/MetaManifold/Scaling.agda | 244 ++++++++ proofs/agda/README.md | 140 +++++ proofs/agda/metamanifold.agda-lib | 6 + proofs/tests/axiom-audit.sh | 141 +++++ renv.lock | 98 +++ src/analysis/analysis.jl | 241 +++++++- src/analysis/differential.jl | 492 ++++++++++++--- src/core/validate.jl | 82 ++- src/server/routes/analysis.jl | 11 +- src/server/routes/config.jl | 41 +- src/server/routes/databases.jl | 15 +- src/server/routes/results.jl | 10 +- test/runtests.jl | 2 + test/unit/test_differential.jl | 379 +++++++++++- test/unit/test_install_pins.jl | 57 +- test/unit/test_permanova.jl | 103 ++++ test/unit/test_r_runtime.jl | 100 +++ test/unit/test_routes.jl | 10 +- test/unit/test_write_gates.jl | 138 +++++ 42 files changed, 3406 insertions(+), 253 deletions(-) create mode 100644 frontend/src/components/annotationShared.test.ts create mode 100644 frontend/src/components/sharedRanks.test.ts create mode 100644 frontend/src/components/sharedRanks.ts create mode 100644 frontend/src/hooks/useApi.test.ts create mode 100644 proofs/agda/MetaManifold/All.agda create mode 100644 proofs/agda/MetaManifold/BenjaminiHochberg.agda create mode 100644 proofs/agda/MetaManifold/Prelude.agda create mode 100644 proofs/agda/MetaManifold/Scaling.agda create mode 100644 proofs/agda/README.md create mode 100644 proofs/agda/metamanifold.agda-lib create mode 100755 proofs/tests/axiom-audit.sh create mode 100644 test/unit/test_permanova.jl create mode 100644 test/unit/test_write_gates.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 912daaba..10cf5483 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,33 @@ concurrency: jobs: test: - name: Julia ${{ matrix.julia-version }} / ${{ matrix.os }} - runs-on: ${{ matrix.os }} + # The name is a fixed string, it carries no version, and the job has NO matrix. + # Both are required, and the first without the second is a trap that has already + # been sprung here once. + # + # A branch ruleset matches a required status check by the DISPLAY NAME GitHub posts + # for the job. A renamed check does not fail -- it is simply absent, and an absent + # required check can never be satisfied, so every pull request deadlocks until an + # admin bypasses the rule, triggered by nothing more alarming than editing a version + # number. + # + # MEASURED on hyperpolymath/MetaManifold-WebUI#39 (2026-09-21): this job, named exactly `Julia tests` but still + # carrying a 1x1 `strategy.matrix`, posted `Julia tests (1.12.5, ubuntu-24.04)`. + # GitHub appends the matrix combination whenever the name does not already reference + # the matrix, so a 1x1 matrix is still a matrix and the pins were still embedded in + # the check name. The matrix was therefore removed rather than merely renamed around: + # it selected exactly one combination and bought nothing. + # + # The two values it held are literals below, each keeping its own reasoning. + # `test/unit/test_install_pins.jl` asserts, for EVERY job in this file, that the name + # interpolates nothing AND that the job has no matrix -- so this cannot regress + # silently. If a matrix is ever genuinely wanted here, the required context must move + # to a matrix-free aggregator job first. + name: Julia tests + # Explicit, not ubuntu-latest: the R pin below names an apt package revision built + # for 24.04, and a runner that silently rolled to the next LTS would take that pin + # with it. `runs-on` cannot read `env`, so this is a literal. + runs-on: ubuntu-24.04 # The repository's root .Rprofile sources renv/activate.R, so R started from # the checkout auto-activates renv and rewrites the library paths. renv is a # local-development convenience; CI installs into the system library and @@ -28,19 +53,6 @@ jobs: # the environment, so those commands pass --no-init-file instead. env: RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" - strategy: - fail-fast: false - matrix: - # Pinned rather than tracking latest stable, because a pinned Manifest.toml - # resolved by an unpinned Julia is not a reproducible build. This must equal - # julia_version in Manifest.toml; it cannot be read from the pin file, since - # nothing can be read before Julia exists. test/unit/test_install_pins.jl - # fails if the three ever disagree. - julia-version: ["1.12.5"] - # Explicit, not ubuntu-latest: the R pin below names an apt package revision - # built for 24.04, and a runner that silently rolled to the next LTS would - # take that pin with it. - os: [ubuntu-24.04] steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -48,7 +60,12 @@ jobs: - name: Set up Julia uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 with: - version: ${{ matrix.julia-version }} + # Pinned rather than tracking latest stable, because a pinned Manifest.toml + # resolved by an unpinned Julia is not a reproducible build. This must equal + # julia_version in Manifest.toml; it cannot be read from the pin file, since + # nothing can be read before Julia exists. test/unit/test_install_pins.jl + # fails if the three ever disagree. + version: "1.12.5" - name: Cache Julia packages uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0 @@ -139,7 +156,7 @@ jobs: sudo chmod 777 "$RENV_PATHS_CACHE" sudo env R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'install.packages(c("renv", "BiocManager"), repos="https://cloud.r-project.org", lib=.Library)' sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)' - Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","dplyr","MASS")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' + Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","dplyr","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' # Saved on failure too, since the packages built before a failure are the ones # the next attempt should reuse. Root writes the cache, so it is made readable first. @@ -230,7 +247,7 @@ jobs: julia --project=. -e ' using RCall R"cat(.libPaths(), sep=\"\n\")" - for pkg in ["dada2", "dplyr", "vegan", "tibble", "MASS"] + for pkg in ["dada2", "dplyr", "vegan", "tibble", "MASS", "zCompositions"] R"if (!require($pkg, character.only=TRUE, quietly=TRUE)) stop($pkg, \" not found\")" end ' @@ -282,3 +299,45 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} slug: JoshuaJewell/MetaManifold-WebUI fail_ci_if_error: false + + # Machine-checked proofs of the Benjamini-Hochberg adjustment and the TSS/RLE + # scaling algebra (proofs/agda, proofs/agda/README.md). + # Agda publishes no 2.6.4.3 binary, and agda-stdlib 2.1 requires Agda 2.6.4.x, + # so the pair comes from Debian trixie, which packages exactly that pair with + # pre-built library interfaces. The image is pinned by digest and the two + # packages by revision, so a rebuild in the archive fails loudly here instead + # of silently changing the checker. These pins live here rather than in + # config/defaults/tool_versions.yml because no other job and no installer + # reads them. + agda: + name: Agda proofs + runs-on: ubuntu-24.04 + container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb + timeout-minutes: 30 + env: + DEBIAN_FRONTEND: noninteractive + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Install Agda and the standard library + run: | + apt-get update -qq + apt-get install -y --no-install-recommends agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 + agda --version | grep -qx 'Agda version 2.6.4.3' + grep -qx 'name: standard-library-2.1' /usr/share/agda-stdlib/standard-library.agda-lib + + # Runs before the type-check: --safe already refuses postulates, but the + # audit also refuses holes, check-disabling pragmas and flags, and any + # module that MetaManifold/All.agda does not import (an unimported module + # is never type-checked at all). + - name: Axiom audit + run: bash proofs/tests/axiom-audit.sh + + - name: Type-check + env: + AGDA_DIR: /tmp/agda-dir + run: | + mkdir -p "$AGDA_DIR" + echo /usr/share/agda-stdlib/standard-library.agda-lib > "$AGDA_DIR/libraries" + cd proofs/agda + agda MetaManifold/All.agda diff --git a/.gitignore b/.gitignore index 1436ebab..42d6c9c7 100644 --- a/.gitignore +++ b/.gitignore @@ -336,6 +336,8 @@ LocalPreferences.toml # Development server log (dev/dev.sh) dev/server.log +# Agda interface files (proofs/agda) +proofs/agda/_build/ # Proof siblings fetched by scripts/fetch-siblings.sh, and Agda interface files /_build/ *.agdai diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 504ad6b0..930ec561 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,7 +98,7 @@ A pull request does not reopen these. An issue can discuss them. - `config/defaults/tool_versions.yml` pins Julia, R, bun and the external tools, and `install.jl` and CI both install from it. The Julia version is repeated in - the CI matrix. Change a pin by editing that file, and expect the change to be + the CI workflow. Change a pin by editing that file, and expect the change to be discussed. - R packages are pinned in `renv.lock`. - CI job names carry no version numbers, so that a pin change leaves required diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 6f50ed68..8181ae6d 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -347,13 +347,27 @@ analysis: # comparing two conditions on the raw read counts, with Benjamini-Hochberg # adjusted p-values. differential: + # The model fitted to each taxon. nb_glm: a negative-binomial GLM on the read + # counts with a library-size offset; the effect is a log2 fold change. clr_welch: + # Welch's t-test on centred log-ratios over the taxa passing min_prevalence, + # after zCompositions::cmultRepl replaces their zeros; the effect is a + # difference in CLR units, not a fold change. + method: nb_glm # How library size enters each model, as a log offset: tss (total reads per # sample) or rle (median-of-ratios over the taxa present in every sample). # rle is refused when no taxon is present in every sample. + # Used by nb_glm only; clr_welch ignores it, since log-ratios carry no library size. offset: tss # Fraction of samples, 0 to 1, in which a taxon must have reads to be - # tested. Taxa below it are listed as filtered, without a p-value. + # tested. Taxa below it are listed as filtered, without a p-value. For + # clr_welch it is applied before zero replacement, so on a sparse table it also + # keeps imputed values for rare taxa out of every sample; clr_welch refuses a + # taxon seen in fewer than 2 samples. min_prevalence: 0.0 + # clr_welch only: cmultRepl's frac, between 0 and 1. An imputed proportion above + # its taxon's smallest observed proportion is lowered to this fraction of it. + # A replaced zero is not a measurement. + replacement_delta: 0.65 ## Occupancy probe (MMW-01) # Measures each attested stage's memory: Julia's allocation total, and the diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 10410bd9..87457dad 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -243,15 +243,32 @@ export interface ComparisonRequest extends AnalysisRequest { aggregate?: boolean } +export interface PermanovaTerm { + term: string + r2: number + f_statistic: number | null + /** Null when the term could not be tested; `untestable_reason` says why. */ + p_value: number | null + /** Set when within-block permutation cannot change this term. */ + untestable_reason?: string | null +} + export interface PermanovaResult { text: string + /** The first term (group, when it varies); r2, f_statistic and p_value are its own. */ + term?: string r2: number | null f_statistic: number | null p_value: number | null + untestable_reason?: string | null + /** Permutations actually performed: fewer than 999 when the blocks admit fewer. */ + permutations?: number + /** Smallest p-value attainable with that many permutations. */ + min_p_value?: number /** True when permutations were restricted within individuals. */ blocked?: boolean /** One row per model term, tested sequentially in formula order. */ - terms?: { term: string; r2: number; f_statistic: number | null; p_value: number | null }[] + terms?: PermanovaTerm[] /** PERMDISP on the same distances: do the groups differ in spread? */ dispersion?: PermdispResult | { error: string } } @@ -381,18 +398,32 @@ export interface DifferentialRow { taxon: string status: DifferentialStatus note: string - /** Natural-log fold change, contrast over reference. */ + /** + * nb_glm: natural-log fold change, contrast over reference. clr_welch: mean + * CLR of contrast minus mean CLR of reference, which is not a fold change. + */ estimate: number | null + /** nb_glm only; null for clr_welch. */ log2_fold_change: number | null standard_error: number | null + /** z (nb_glm) or Welch's t (clr_welch). */ statistic: number | null pvalue: number | null /** Benjamini-Hochberg adjusted; null outside the tested family. */ padj: number | null + /** nb_glm only. */ dispersion_theta: number | null prevalence: number } +export type DifferentialMethod = 'nb_glm' | 'clr_welch' + +/** Which row field holds the effect size, and what to call it. */ +export interface DifferentialEffect { + key: 'log2_fold_change' | 'estimate' + label: string +} + export interface DifferentialResult { status: 'ok' | 'partial' method: string @@ -400,9 +431,26 @@ export interface DifferentialResult { table: string groups: { reference: string; contrast: string } n_samples: Record + effect: DifferentialEffect + /** Empty for clr_welch, which takes no library-size offset. */ size_factors: { sample: string; group: string; factor: number }[] - config: { offset: 'tss' | 'rle'; min_prevalence: number } - diagnostics: { n_taxa: number; n_tested: number; n_failed: number; n_boundary: number; n_filtered: number } + config: { + method: DifferentialMethod + min_prevalence: number + /** nb_glm only. */ + offset?: 'tss' | 'rle' + /** clr_welch only. */ + replacement_delta?: number + } + diagnostics: { + n_taxa: number; n_tested: number; n_failed: number; n_boundary: number; n_filtered: number + /** clr_welch only. */ + zero_replacement?: { + method: string; delta: number; zeros_replaced: number; n_taxa_in_composition: number + n_taxa_unobserved: number; median_imputed_fraction: number; max_imputed_fraction: number + warnings: string[] + } + } rows: DifferentialRow[] figure: unknown } diff --git a/frontend/src/components/AnalysisWorkspace.tsx b/frontend/src/components/AnalysisWorkspace.tsx index c8c6b5af..e7295004 100644 --- a/frontend/src/components/AnalysisWorkspace.tsx +++ b/frontend/src/components/AnalysisWorkspace.tsx @@ -3,7 +3,7 @@ import { useEffect, useMemo, useState, type ReactNode } from 'react' import type { AnnotationSource, ComparisonRunSpec } from '../api/types' import { AnalysisSourceContext } from './analysisSource' -import { useSharedResultsTables } from './annotationShared' +import { useSharedResultsTablesState } from './annotationShared' import { DiversityPanel } from './DiversityPanel' import { TaxaCompositionChart } from './TaxaCompositionChart' import { VennPanel } from './VennPanel' @@ -80,7 +80,7 @@ export function AnalysisWorkspace({ study, runs: baseRuns, source, perRun }: { }, [aggregate, selectedRuns]) //## Dataset: results tables shared by every selected run - const options = useSharedResultsTables(study, selectedRuns) + const { options, error: optionsError } = useSharedResultsTablesState(study, selectedRuns) const [analysisKey, setAnalysisKey] = useState(null) useEffect(() => { setAnalysisKey(current => current && options.some(o => o.key === current) ? current : options[0]?.key ?? null) @@ -138,6 +138,8 @@ export function AnalysisWorkspace({ study, runs: baseRuns, source, perRun }: { onChange={e => setAnalysisKey(e.target.value)}> {options.map(o => )} + ) : optionsError ? ( + could not list the results tables: {optionsError} ) : ( no results table shared by the selected runs )} diff --git a/frontend/src/components/CompositionPanel.tsx b/frontend/src/components/CompositionPanel.tsx index 77ad8869..5e677683 100644 --- a/frontend/src/components/CompositionPanel.tsx +++ b/frontend/src/components/CompositionPanel.tsx @@ -16,6 +16,12 @@ import type { // Fixed colour of the catch-all "Unassigned" bucket, mirroring the backend. const UNASSIGNED_COLOUR = '#95a5a6' +/** + * Browse a run's composition by taxonomy rank or category, with sub-group and + * results-table selectors and controls to save category colours. The analysis + * context's annotation source takes precedence over `source`; absent both, + * VSEARCH is used. A failed table lookup clears the options and shows a toast. + */ export function CompositionPanel({ study, run, group, subgroups, source: configuredSource, }: { @@ -53,6 +59,10 @@ export function CompositionPanel({ setTableOptions(opts) setTable(current => opts.some(o => o.key === current) ? current : opts.some(o => o.key === 'merged') ? 'merged' : opts[0]?.key ?? current) + }).catch(err => { + if (cancelled) return + setTableOptions([]) + toast.error(`Could not list the results tables: ${errorMessage(err)}`) }) return () => { cancelled = true } }, [study, run, group]) diff --git a/frontend/src/components/DifferentialPanel.tsx b/frontend/src/components/DifferentialPanel.tsx index ff15b507..8f78e326 100644 --- a/frontend/src/components/DifferentialPanel.tsx +++ b/frontend/src/components/DifferentialPanel.tsx @@ -1,16 +1,16 @@ // SPDX-License-Identifier: AGPL-3.0-only // SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { api } from '../api/client' import { errorMessage } from '../api/errorMessage' import { useToast } from './Toast' import type { ComparisonRunSpec, DifferentialResult } from '../api/types' -import { uniqueRuns } from '../api/types' +import { sharedRanks } from './sharedRanks' import type { AnalysisOption } from './annotationShared' import { AnalysisChart } from './AnalysisChart' import { AddToReport } from './AddToReport' import { saveBlob } from '../utils/download' -import { differentialCsv, formatStat } from './differentialTable' +import { differentialCsv, effectValue, formatStat } from './differentialTable' const cell = { padding: '3px 10px', textAlign: 'right' } as const const head = { padding: '4px 10px', textAlign: 'right' } as const @@ -21,10 +21,10 @@ function csvBlob(result: DifferentialResult): Blob { } /** - * Differential abundance between exactly two runs or sub-groups: a - * negative-binomial model per taxon at the chosen rank, with BH-adjusted - * p-values, shown as a volcano plot and a table that can be downloaded or - * added to the report. The first selected condition is the reference. + * Differential abundance between exactly two runs or sub-groups: a model per + * taxon at the chosen rank (negative-binomial GLM or Welch's t-test on CLR values, as + * configured), with BH-adjusted p-values, shown as a volcano plot and a table + * that can be downloaded or added to the report. The first selected condition is the reference. */ export function DifferentialPanel({ study, runs, option, aggregate }: { study: string @@ -36,42 +36,58 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { const [ranks, setRanks] = useState([]) const [rank, setRank] = useState('') const [ranksReady, setRanksReady] = useState(false) + const [ranksError, setRanksError] = useState(null) const [result, setResult] = useState(null) const [loading, setLoading] = useState(false) + // Only the response to the latest request, for the current inputs, is shown. + const request = useRef(0) // The ranks shared by every selected run. useEffect(() => { + setRanksError(null) if (!option || runs.length === 0) { setRanks([]); setRank(''); setRanksReady(false); return } let cancelled = false setRanksReady(false) - Promise.all( - uniqueRuns(runs).map(r => - api.analysis.ranks(study, r.run, { - table: option.table, - group: r.group, - ...(r.source ? { source: r.source } : {}), - }).catch(() => [] as string[]) - ) - ).then(results => { + sharedRanks(runs, r => api.analysis.ranks(study, r.run, { + table: option.table, + group: r.group, + ...(r.source ? { source: r.source } : {}), + })).then(shared => { if (cancelled) return - const shared = results.reduce((acc, cur) => acc.filter(r => cur.includes(r)), results[0] ?? []) setRanks(shared) setRanksReady(true) setRank(current => shared.includes(current) ? current : (shared[shared.length - 1] ?? '')) + }).catch(err => { + if (cancelled) return + setRanks([]) + setRank('') + setRanksError(errorMessage(err)) }) return () => { cancelled = true } }, [study, runs, option]) + // A result answers the inputs it was computed from; new inputs discard it, + // along with any request still in flight for the old ones. + useEffect(() => { + request.current++ + setResult(null) + setLoading(false) + }, [study, runs, option?.table, rank, aggregate]) + + /** Fit the model for the current inputs and show it, unless the inputs changed meanwhile. */ const run = async () => { if (!option || !rank) return + const req = ++request.current setLoading(true) try { - setResult(await api.analysis.differential(study, { runs, table: option.table, rank, aggregate })) + const res = await api.analysis.differential(study, { runs, table: option.table, rank, aggregate }) + if (req === request.current) setResult(res) } catch (err) { + if (req !== request.current) return setResult(null) toast.error(`Differential abundance failed: ${errorMessage(err)}`) } finally { - setLoading(false) + if (req === request.current) setLoading(false) } } @@ -94,9 +110,12 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { {ranksReady && ranks.length === 0 && ( No taxonomy rank shared by the selected runs. )} + {ranksError && ( + Could not read the taxonomy ranks: {ranksError} + )} {runs.length === 2 && ( - Reference: {runs[0].prefix ?? runs[0].run}; positive fold changes mean more abundant in {runs[1].prefix ?? runs[1].run}. + Reference: {runs[0].prefix ?? runs[0].run}; a positive effect means more abundant in {runs[1].prefix ?? runs[1].run}. )} @@ -109,6 +128,12 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { {result.diagnostics.n_boundary > 0 && `, ${result.diagnostics.n_boundary} at a dispersion boundary`} {result.diagnostics.n_filtered > 0 && `, ${result.diagnostics.n_filtered} below the prevalence threshold`} . Samples: {Object.entries(result.n_samples).map(([g, n]) => `${g} ${n}`).join(', ')}. + {result.diagnostics.zero_replacement && (() => { + const z = result.diagnostics.zero_replacement + return ` ${z.zeros_replaced} zeros replaced by ${z.method} over ${z.n_taxa_in_composition} taxa; ` + + `replaced values hold ${(100 * z.median_imputed_fraction).toPrecision(2)}% of the median sample ` + + `and at most ${(100 * z.max_imputed_fraction).toPrecision(2)}% of any, and are not measurements.` + })()}

@@ -126,8 +151,8 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { {result.rank} - log2 FC - Estimate (ln) + {result.effect.label} + {result.config.method === 'nb_glm' && Estimate (ln)} SE p padj (BH) @@ -138,8 +163,8 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { {result.rows.map(r => ( {r.taxon} - {formatStat(r.log2_fold_change)} - {formatStat(r.estimate)} + {formatStat(effectValue(result, r))} + {result.config.method === 'nb_glm' && {formatStat(r.estimate)}} {formatStat(r.standard_error)} {formatStat(r.pvalue)} {formatStat(r.padj)} diff --git a/frontend/src/components/DiversityPanel.tsx b/frontend/src/components/DiversityPanel.tsx index b3d16dd2..ae84d450 100644 --- a/frontend/src/components/DiversityPanel.tsx +++ b/frontend/src/components/DiversityPanel.tsx @@ -1,6 +1,6 @@ // © 2026 Joshua Benjamin Jewell. All rights reserved. // Licensed under the GNU Affero General Public License version 3 (AGPLv3). -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { api } from '../api/client' import { AnalysisChart } from './AnalysisChart' import { useAlphaMetricSelection, AlphaMetricToggles, ALPHA_METRICS, extractAlphaPanel } from './alphaMetrics' @@ -22,11 +22,27 @@ export function DiversityPanel({ study, runs, option, aggregate }: { const [permanova, setPermanova] = useState(null) const [loading, setLoading] = useState(null) const [rAvailable, setRAvailable] = useState(null) + const [capabilitiesError, setCapabilitiesError] = useState(null) + // Only the response to the latest request, for the current inputs, is shown. + const request = useRef(0) + // A failed check is reported as such, not as "R is unavailable". useEffect(() => { - api.analysis.capabilities().then(c => setRAvailable(c.r_available)).catch(() => setRAvailable(false)) + api.analysis.capabilities() + .then(c => setRAvailable(c.r_available)) + .catch(err => setCapabilitiesError(errorMessage(err))) }, []) + // Results answer the inputs they were computed from; new inputs discard them, + // along with any request still in flight for the old ones. + useEffect(() => { + request.current++ + setAlphaFig(null) + setNmdsFig(null) + setPermanova(null) + setLoading(null) + }, [study, runs, option?.table, aggregate]) + const body = option && runs.length >= 2 ? { table: option.table, @@ -36,22 +52,33 @@ export function DiversityPanel({ study, runs, option, aggregate }: { } : null + /** Compute one analysis for the current inputs and show it, unless the inputs changed meanwhile. */ const run = async (type: 'alpha' | 'nmds' | 'permanova') => { if (!body) return + const req = ++request.current + const current = () => req === request.current setLoading(type) try { if (type === 'alpha') { - setAlphaFig(await api.analysis.compareAlpha(study, body)) + const fig = await api.analysis.compareAlpha(study, body) + if (current()) setAlphaFig(fig) } else if (type === 'nmds') { - setNmdsFig(await api.analysis.nmds(study, body)) + const fig = await api.analysis.nmds(study, body) + if (current()) setNmdsFig(fig) } else { - setPermanova(await api.analysis.permanova(study, body)) + const res = await api.analysis.permanova(study, body) + if (current()) setPermanova(res) } } catch (err) { + if (!current()) return + // A failed rerun must not leave the previous result looking current. + if (type === 'alpha') setAlphaFig(null) + else if (type === 'nmds') setNmdsFig(null) + else setPermanova(null) const label = type === 'alpha' ? 'Alpha comparison' : type === 'nmds' ? 'NMDS' : 'PERMANOVA' toast.error(`${label} failed: ${errorMessage(err)}`) } finally { - setLoading(null) + if (current()) setLoading(null) } } @@ -82,6 +109,11 @@ export function DiversityPanel({ study, runs, option, aggregate }: { NMDS and PERMANOVA need R with vegan. )} + {capabilitiesError && ( + + Could not check whether R is available: {capabilitiesError} + + )} {alphaFig != null && ( @@ -99,19 +131,20 @@ export function DiversityPanel({ study, runs, option, aggregate }: {
PERMANOVA Results
{permanova.text}
- {(permanova.terms?.length - ? permanova.terms - : permanova.p_value != null - ? [{ term: 'model', r2: permanova.r2 ?? NaN, f_statistic: permanova.f_statistic, p_value: permanova.p_value }] - : [] - ).map(t => ( + {(permanova.terms ?? []).map(t => (

- {t.term}: R² = {t.r2.toFixed(3)} · F = {t.f_statistic?.toFixed(2) ?? 'n/a'} · p = {t.p_value?.toFixed(4) ?? 'n/a'} + {t.term}: R² = {t.r2.toFixed(3)} · F = {t.f_statistic?.toFixed(2) ?? 'n/a'} ·{' '} + {t.untestable_reason ? `p not reported: ${t.untestable_reason}` : `p = ${t.p_value?.toFixed(4) ?? 'n/a'}`}

))} {(permanova.terms?.length ?? 0) > 1 && (

Terms are tested in order, each after the ones above it.

)} + {permanova.permutations != null && permanova.min_p_value != null && ( +

+ {permanova.permutations} permutations; the smallest attainable p is {permanova.min_p_value.toFixed(4)}. +

+ )} {permanova.blocked && (

Permutations restricted within individuals.

)} diff --git a/frontend/src/components/PipelineStages.tsx b/frontend/src/components/PipelineStages.tsx index 8a96ab41..85129267 100644 --- a/frontend/src/components/PipelineStages.tsx +++ b/frontend/src/components/PipelineStages.tsx @@ -150,8 +150,10 @@ export const CONFIG_DESCRIPTIONS: Record = { 'analysis.alpha.rarefaction_iterations': 'rarefy draws averaged for each alpha diversity value.', 'analysis.beta.normalisation': 'Preparation before the Bray-Curtis dissimilarity used by NMDS and PERMANOVA: hellinger (square root of relative abundances), none, rarefy or srs.', 'analysis.beta.depth': 'Reads per sample for rarefy or srs; 0 uses the smallest non-empty sample.', - 'analysis.differential.offset': 'Library-size offset in each negative-binomial model: tss (total reads per sample) or rle (median-of-ratios over taxa present in every sample; refused when there are none).', + 'analysis.differential.method': 'Model fitted to each taxon: nb_glm (negative-binomial GLM on read counts; effect is a log2 fold change) or clr_welch (Welch\'s t-test on centred log-ratios over the taxa passing min_prevalence, after zCompositions::cmultRepl zero replacement; effect is a CLR difference, not a fold change).', + 'analysis.differential.offset': 'Library-size offset in each negative-binomial model: tss (total reads per sample) or rle (median-of-ratios over taxa present in every sample; refused when there are none). Used by nb_glm only.', 'analysis.differential.min_prevalence': 'Fraction of samples, 0 to 1, in which a taxon must have reads to be tested; taxa below it are listed as filtered.', + 'analysis.differential.replacement_delta': 'clr_welch only: cmultRepl\'s frac, between 0 and 1; an imputed proportion above its taxon\'s smallest observed proportion is lowered to this fraction of it. Replaced zeros are not measurements.', 'analysis.alpha.show_points': 'Show individual samples overlaid on alpha comparison boxplots.', 'analysis.alpha.annotate_significance': 'Annotate alpha comparison panels with Kruskal-Wallis significance.', 'analysis.alpha.pairwise_brackets': 'Run BH-adjusted pairwise Wilcoxon rank-sum tests between every group pair and draw brackets, including n.s. results.', @@ -248,8 +250,10 @@ export const CONFIG_TYPES: Record = { 'analysis.alpha.rarefaction_iterations': { kind: 'int' }, 'analysis.beta.normalisation': { kind: 'enum', options: ['hellinger', 'none', 'rarefy', 'srs'] }, 'analysis.beta.depth': { kind: 'int' }, + 'analysis.differential.method': { kind: 'enum', options: ['nb_glm', 'clr_welch'] }, 'analysis.differential.offset': { kind: 'enum', options: ['tss', 'rle'] }, 'analysis.differential.min_prevalence': { kind: 'float', min: 0, max: 1, step: 0.05 }, + 'analysis.differential.replacement_delta': { kind: 'float', min: 0.01, max: 0.99, step: 0.05 }, 'analysis.alpha.show_points': { kind: 'boolean' }, 'analysis.alpha.annotate_significance': { kind: 'boolean' }, 'analysis.alpha.pairwise_brackets': { kind: 'boolean' }, diff --git a/frontend/src/components/TaxaCompositionChart.tsx b/frontend/src/components/TaxaCompositionChart.tsx index 2b8221a4..9601aebd 100644 --- a/frontend/src/components/TaxaCompositionChart.tsx +++ b/frontend/src/components/TaxaCompositionChart.tsx @@ -1,6 +1,6 @@ // (c) 2026 Joshua Benjamin Jewell. All rights reserved. // Licensed under the GNU Affero General Public License version 3 (AGPLv3). -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { api } from '../api/client' import { useAnalysisSource } from './analysisSource' import { errorMessage } from '../api/errorMessage' @@ -24,6 +24,13 @@ const LABEL_STYLE: React.CSSProperties = { gap: 4, } +/** + * Select and compute a taxon or category bar chart for one run, or for `runs` + * when that list is non-empty, with optional facets across runs. `table` + * defaults to merged. Providing `subgroup`, including null for all sub-groups, + * controls that selection and hides the internal selector; facets set their + * own scope. Changed inputs clear the figure; chart failures show a toast. + */ export function TaxaCompositionChart({ study, run, @@ -78,6 +85,8 @@ export function TaxaCompositionChart({ //## Chart state const [figure, setFigure] = useState(null) const [loading, setLoading] = useState(false) + // Only the response to the latest request, for the current inputs, is shown. + const request = useRef(0) //## Derived flags const isCrossRun = runs !== undefined && runs.length > 0 @@ -102,9 +111,11 @@ export function TaxaCompositionChart({ useEffect(() => { if (tag !== 'rank') return if (!refRun) { setRanks([]); return } + let cancelled = false api.analysis .ranks(study, refRun, { table: effectiveTable, group: refGroup ?? undefined }) .then(result => { + if (cancelled) return setRanks(result) setRank(current => current && result.includes(current) @@ -112,7 +123,8 @@ export function TaxaCompositionChart({ : result[result.length - 1] ?? null ) }) - .catch(() => setRanks([])) + .catch(() => { if (!cancelled) setRanks([]) }) + return () => { cancelled = true } }, [study, refRun, refGroup, tag, effectiveTable]) //## Fetch category sets when tag='category' @@ -131,7 +143,22 @@ export function TaxaCompositionChart({ const gridRows = (figure as { layout?: { grid?: { rows?: number } } } | null)?.layout?.grid?.rows ?? 1 + // Everything a request is built from. A figure answers the inputs it was + // computed from, so new inputs discard it, along with any request still in + // flight for the old ones. Keyed on content so a new-but-equal `runs` array + // does not clear the chart. + const inputsKey = JSON.stringify([ + study, run, group, runs, effectiveTable, source, tag, rank, catSet, topN, + relative, mode, keepEmpty, isFaceted, facetRows, facetCols, subgroup, + ]) + useEffect(() => { + request.current++ + setFigure(null) + setLoading(false) + }, [inputsKey]) + //## Compute handler + /** Build the chart for the current inputs and show it, unless the inputs changed meanwhile. */ const handleShow = async () => { const value = tag === 'rank' ? rank : catSet if (!value) return @@ -140,6 +167,7 @@ export function TaxaCompositionChart({ toast.error('No run specified') return } + const req = ++request.current setLoading(true) try { const body = { @@ -151,27 +179,31 @@ export function TaxaCompositionChart({ keep_empty: keepEmpty, ...(tag === 'rank' ? { top_n: topN } : {}), } + let fig: unknown if (isFaceted) { - setFigure(await api.analysis.chartFacet(study, { + fig = await api.analysis.chartFacet(study, { ...body, runs: runs!, rows: facetRows, cols: facetCols, - })) + }) } else if (isCrossRun) { - setFigure(await api.analysis.chartCompare(study, { + fig = await api.analysis.chartCompare(study, { ...body, subgroup: subgroup ?? null, runs: runs!, - })) + }) } else { - setFigure(await api.analysis.chart(study, run!, - { ...body, subgroup: subgroup ?? null, ...(source ? { source } : {}) }, group)) + fig = await api.analysis.chart(study, run!, + { ...body, subgroup: subgroup ?? null, ...(source ? { source } : {}) }, group) } + if (req === request.current) setFigure(fig) } catch (err) { + if (req !== request.current) return + setFigure(null) toast.error(`Chart failed: ${errorMessage(err)}`) } finally { - setLoading(false) + if (req === request.current) setLoading(false) } } diff --git a/frontend/src/components/VennPanel.tsx b/frontend/src/components/VennPanel.tsx index e276f24d..4ed37a0f 100644 --- a/frontend/src/components/VennPanel.tsx +++ b/frontend/src/components/VennPanel.tsx @@ -8,11 +8,12 @@ import { api } from '../api/client' import { errorMessage } from '../api/errorMessage' import { useToast } from './Toast' import type { ComparisonRunSpec, VennResult } from '../api/types' -import { uniqueRuns } from '../api/types' +import { sharedRanks } from './sharedRanks' import type { AnalysisOption } from './annotationShared' type Mode = 'euler' | 'upset' +/** Taxon overlap between the selected runs at one shared rank, as an Euler or UpSet diagram. */ export function VennPanel({ study, runs, option }: { study: string runs: ComparisonRunSpec[] @@ -25,6 +26,9 @@ export function VennPanel({ study, runs, option }: { const [result, setResult] = useState(null) const [loading, setLoading] = useState(false) const [ranksReady, setRanksReady] = useState(false) + const [ranksError, setRanksError] = useState(null) + // Only the response to the latest request, for the current inputs, is shown. + const request = useRef(0) const box = useRef(null) const [width, setWidth] = useState(580) @@ -38,35 +42,43 @@ export function VennPanel({ study, runs, option }: { // Discover the intersection of taxonomy ranks available across all selected runs. useEffect(() => { + setRanksError(null) if (!option || runs.length === 0) { setRanks([]); setRank(''); setRanksReady(false); return } let cancelled = false setRanksReady(false) - Promise.all( - uniqueRuns(runs).map(r => - api.analysis.ranks(study, r.run, { - table: option.table, - group: r.group, - ...(r.source ? { source: r.source } : {}), - }).catch(() => [] as string[]) - ) - ).then(results => { + sharedRanks(runs, r => api.analysis.ranks(study, r.run, { + table: option.table, + group: r.group, + ...(r.source ? { source: r.source } : {}), + })).then(intersection => { if (cancelled) return - const intersection = results.reduce((acc, cur) => - acc.filter(r => cur.includes(r)), - results[0] ?? [] - ) setRanks(intersection) setRanksReady(true) // Default to the deepest (last) shared rank. setRank(current => intersection.includes(current) ? current : (intersection[intersection.length - 1] ?? '') ) + }).catch(err => { + if (cancelled) return + setRanks([]) + setRank('') + setRanksError(errorMessage(err)) }) return () => { cancelled = true } }, [study, runs, option]) + // A diagram answers the inputs it was computed from; new inputs discard it, + // along with any request still in flight for the old ones. + useEffect(() => { + request.current++ + setResult(null) + setLoading(false) + }, [study, runs, option?.table, rank]) + + /** Compute the overlap for the current inputs and show it, unless the inputs changed meanwhile. */ const runVenn = async () => { if (!option || !rank) return + const req = ++request.current setLoading(true) try { const res = await api.analysis.venn(study, { @@ -74,11 +86,13 @@ export function VennPanel({ study, runs, option }: { table: option.table, rank, }) - setResult(res) + if (req === request.current) setResult(res) } catch (err) { + if (req !== request.current) return + setResult(null) toast.error(`Taxon overlap failed: ${errorMessage(err)}`) } finally { - setLoading(false) + if (req === request.current) setLoading(false) } } @@ -111,6 +125,9 @@ export function VennPanel({ study, runs, option }: { {ranksReady && ranks.length === 0 && ( No taxonomy rank shared by the selected runs. )} + {ranksError && ( + Could not read the taxonomy ranks: {ranksError} + )}