From 1f05eefcaa6306236cbeeed2c844e007e705dba8 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:44:23 +0000 Subject: [PATCH 1/3] fix(test): probe R once so the guard answers what the consumer asks The live-R guard asked a different question from the record it stood in front of: it probed an empty package list, its consumer probed the four in R_PACKAGES, so on any checkout where R answers but the packages are absent -- every bare checkout -- the guard reported ready and the consumer threw outside any try, erroring the testset before its first @test (#43). The @info branch could not fire for the case it was written for. provenance.jl now answers the question once. RProbeStatus carries a status (:ok, :packages_missing, :r_unreachable, :no_lockfile) beside the record, and r_probe_status performs the single probe whose record the suite then asserts on, so guard and consumer cannot diverge. The failure's component names travel as a ProbeFailure field rather than as prose, so classification is a field read, and probe_r collects every absent package instead of stopping at the first: four missing packages report four. The skip is loud and named: the testset is titled with the status and the reason -- "r - not probed (packages_missing): ... dada2, Biostrings, ShortRead, vegan" -- and recorded with @test_skip, so it lands in the summary's own Broken column rather than in an @info that scrolls past. On CI, where ci.yml installs R and restores all 79 packages from renv.lock, the same block is a hard assertion: a skip there is a provisioning regression, not a bare checkout. The negative control is pure and runs everywhere the live probe skips: :r_unreachable, :packages_missing and :no_lockfile are asserted to be distinguishable in both symbol and text. Verified against the real module on a box with R 4.5.0 and none of the four packages: the :packages_missing path is exercised live, names all four, and records Broken rather than passing or erroring. Closes #43 --- src/core/provenance.jl | 102 ++++++++++++++++++++++++++++++- test/unit/test_provenance.jl | 115 ++++++++++++++++++++++++++++------- 2 files changed, 192 insertions(+), 25 deletions(-) diff --git a/src/core/provenance.jl b/src/core/provenance.jl index 6e8add60..84601ae5 100644 --- a/src/core/provenance.jl +++ b/src/core/provenance.jl @@ -33,6 +33,7 @@ using ..RRuntime: with_r_lock export ProbeFailure, DatabaseReleaseMismatch, ToolProbe, TOOL_PROBES, ToolRecord, JuliaRecord, RRecord, + RProbeStatus, r_probe_ok, r_probe_status, classify_r_probe_failure, DatabaseFormatSpec, DatabaseFormatRecord, DatabaseRecord, CapturedEnvironment, Attestation, probe_tool, probe_julia, probe_r, probe_database, probe_host, probe_metamanifold, @@ -50,8 +51,18 @@ const SCHEMA_VERSION = 1 struct ProbeFailure <: Exception component :: String reason :: String + # Names carried separately from `reason` so a caller can act on WHICH packages + # were absent instead of matching text. Empty for every probe that does not ask + # about packages at all, and for failures that are not about packages: a + # component can be unreachable for reasons that have nothing to do with what is + # installed. `missing_packages` is what makes "R is not installed" and "R is + # installed, dada2 is not" two different answers (#43) rather than one message. + missing_packages :: Vector{String} end +ProbeFailure(component::AbstractString, reason::AbstractString) = + ProbeFailure(String(component), String(reason), String[]) + Base.showerror(io::IO, e::ProbeFailure) = print(io, "provenance: could not prove '$(e.component)': $(e.reason)") @@ -331,22 +342,107 @@ function probe_r(; packages::AbstractVector{<:AbstractString} = R_PACKAGES, with_r_lock(; timeout) do r_version = RCall.rcopy(String, RCall.reval("R.version.string")) found = OrderedDict{String,String}() + missing = String[] for pkg in packages raw = RCall.rcopy(RCall.reval(r_version_query(pkg))) - raw isa AbstractString || throw(ProbeFailure("r", - "package '$pkg' is not installed in the R library in use")) - found[String(pkg)] = String(raw) + # `NA_character_` rather than an exception is how r_version_query + # reports a package DESCRIPTION cannot be read. Collect and carry on: + # a caller told that one package is absent fixes one package, and a + # checkout missing four reports four (#43). + if raw isa AbstractString + found[String(pkg)] = String(raw) + else + push!(missing, String(pkg)) + end end + isempty(missing) || throw(ProbeFailure("r", + "R is reachable but these required packages are not installed: " * + join(missing, ", "), missing)) (r_version, found) end catch err err isa ProbeFailure && rethrow() + # R itself is the component that could not be proved here, so nothing is + # claimed about packages: the interpreter never answered the question. throw(ProbeFailure("r", sprint(showerror, err))) end return RRecord(version, file_sha256(renv_lock), versions) end +""" + RProbeStatus + +What an R probe found, as a status rather than as a value or an exception. + +The question a provenance guard has to answer is not "is R installed" but "may I +read an R record". Those differ whenever the interpreter answers and the packages +do not, which is every bare checkout: the guard used to probe with an empty +package list, its consumer probed `R_PACKAGES`, and so the guard reported ready +exactly where the consumer could only throw (#43). One probe, one answer, one +place to read it from. + +`status` is `:ok` only when `record` may be used. The other three are distinct +because they need different fixes: `:no_lockfile` is a broken checkout, +`:r_unreachable` is a missing toolchain, `:packages_missing` is a half-restored +renv library. `missing` names the packages in the third case and is empty +otherwise. +""" +struct RProbeStatus + status :: Symbol + record :: Union{RRecord,Nothing} + missing :: Vector{String} + reason :: String +end + +""" + r_probe_ok(s::RProbeStatus) -> Bool + +Whether `s` holds an R record that may be read. False means no R record exists, so +neither the version nor any package version may be presented as a fact. +""" +r_probe_ok(s::RProbeStatus) = s.status === :ok + +""" + classify_r_probe_failure(err::ProbeFailure) -> RProbeStatus + +The status a failed R probe deserves. Pure: no R, no filesystem, no I/O. + +Kept separate from `r_probe_status` so that "R is unreachable" and "R is reachable +but the packages are absent" can be told apart by a test on a machine that has no +R at all (#43, where the whole point is that these carry different messages to +whoever is reading a skip). +""" +classify_r_probe_failure(err::ProbeFailure) = + isempty(err.missing_packages) ? + RProbeStatus(:r_unreachable, nothing, String[], err.reason) : + RProbeStatus(:packages_missing, nothing, copy(err.missing_packages), err.reason) + +""" + r_probe_status(; packages=R_PACKAGES, renv_lock, timeout=60) -> RProbeStatus + +Probe R once and report the outcome as a status. + +`packages` defaults to `R_PACKAGES` and callers should leave it alone: a readiness +check that narrows the list asks a different question from the record it is +standing in front of, which is the defect #43 records. +""" +function r_probe_status(; packages::AbstractVector{<:AbstractString} = R_PACKAGES, + renv_lock::AbstractString = joinpath(repo_root(), "renv.lock"), + timeout::Union{Real,Nothing} = 60) + # Mirrors probe_r's own guard rather than string-matching its message: a missing + # lockfile is this checkout's problem, and reporting it as an unreachable R would + # send whoever reads it to install a toolchain they already have. + isfile(renv_lock) || + return RProbeStatus(:no_lockfile, nothing, String[], "no renv.lock at $renv_lock") + try + return RProbeStatus(:ok, probe_r(; packages, renv_lock, timeout), String[], "") + catch err + err isa ProbeFailure || rethrow() + return classify_r_probe_failure(err) + end +end + ## Reference databases struct DatabaseFormatSpec format :: String diff --git a/test/unit/test_provenance.jl b/test/unit/test_provenance.jl index 42e66205..80a88a34 100644 --- a/test/unit/test_provenance.jl +++ b/test/unit/test_provenance.jl @@ -457,31 +457,102 @@ canned(out::String, err::String = "", code::Int = 0) = end end - r_ready = try - PV.probe_r(; packages = String[], timeout = 30) - true - catch err - @info "Provenance: skipping live R probe" reason=sprint(showerror, err) - false - end - - if r_ready - record = PV.probe_r(; timeout = 120) - @test occursin("R version", record.version) - - ## The bug this exists to prevent. renv.lock is the pin and DESCRIPTION is - # what is loaded; they agree, and both keep R's major.minor-patch - # convention that packageVersion() would flatten. vegan is the witness: - # 2.7-3, never 2.7.3. - lock = read(joinpath(@__DIR__, "..", "..", "renv.lock"), String) - for (pkg, version) in record.packages - @test occursin("\"$pkg\"", lock) - @test occursin("\"Version\": \"$version\"", lock) + ## The R probe (#43). ONE probe answers both questions -- is R reachable, and are + # the packages the pipeline requires installed -- and its record is reused + # rather than asked for twice. Asking twice was the defect: the guard probed an + # empty package list, its consumer probed the four in R_PACKAGES, so on a + # checkout with R and no packages the guard said "ready" and the consumer threw + # outside any try, erroring the testset before its first @test. + r_status = PV.r_probe_status(; timeout = 120) + + if PV.r_probe_ok(r_status) + @testset "r" begin + record = r_status.record + + # The invariant the split guard could not state: the record the suite + # asserts on covers every package the pipeline requires. A narrowed + # probe cannot produce a record that satisfies this. + @test collect(keys(record.packages)) == PV.R_PACKAGES + + @test occursin("R version", record.version) + + ## The bug this exists to prevent. renv.lock is the pin and DESCRIPTION + # is what is loaded; they agree, and both keep R's major.minor-patch + # convention that packageVersion() would flatten. vegan is the witness: + # 2.7-3, never 2.7.3. + lock = read(joinpath(@__DIR__, "..", "..", "renv.lock"), String) + for (pkg, version) in record.packages + @test occursin("\"$pkg\"", lock) + @test occursin("\"Version\": \"$version\"", lock) + end + if haskey(record.packages, "vegan") + @test occursin('-', record.packages["vegan"]) + end end - if haskey(record.packages, "vegan") - @test occursin('-', record.packages["vegan"]) + else + ## Loud, named, and broken rather than passed. The testset title carries + # which world this is and what was missing, so the summary reads + # "packages_missing: dada2, Biostrings, ShortRead, vegan" instead of + # airing an @info that scrolls past (#43 criteria 2 and 3). + @testset "r — not probed ($(r_status.status)): $(r_status.reason)" begin + # ci.yml installs R and restores all 79 packages from renv.lock, so on + # CI a skip here is a provisioning regression, not a bare checkout. The + # failure names the missing packages via this testset's title. + if get(ENV, "CI", "false") == "true" + @test r_status.status === :ok + end + @test_skip PV.probe_r() end end end + ## One probe, not two that can drift (#43, criterion 1). This is a source check + # because the machine it has to hold on is the machine with no R: a skipped + # assertion asserts nothing, and the regression it guards against is a second + # call site, which a runtime check can only observe on a box that has R. + @testset "the R guard and the record it guards are one probe" begin + source = read(joinpath(@__DIR__, "test_provenance.jl"), String) + # Built by concatenation on purpose: written whole, this assertion's own + # source would contain the pattern it forbids and fail on itself. + narrowed = "packages = " * "String[]" + @test !occursin(narrowed, source) + @test occursin("PV.r_probe_status(", source) + end + + ## The negative control (#43, criterion 4): "R is not reachable" and "R is + # reachable but the packages are absent" are different worlds with different + # fixes, and the suite must say which one it is looking at. Pure, so it runs + # wherever the suite runs -- including everywhere the live probe above skips. + @testset "an unreachable R is told apart from a missing package" begin + unreachable = PV.ProbeFailure("r", "RCall: could not find R") + absent_pkgs = PV.ProbeFailure("r", + "R is reachable but these required packages are not installed: dada2, vegan", + ["dada2", "vegan"]) + + unreachable_status = PV.classify_r_probe_failure(unreachable) + @test unreachable_status.status === :r_unreachable + @test isempty(unreachable_status.missing) + @test !PV.r_probe_ok(unreachable_status) + @test occursin("could not find R", unreachable_status.reason) + + absent_status = PV.classify_r_probe_failure(absent_pkgs) + @test absent_status.status === :packages_missing + @test absent_status.missing == ["dada2", "vegan"] + @test !PV.r_probe_ok(absent_status) + # Distinguishable means distinguishable in the text a human reads, not only + # in a symbol a programmer compares. + @test !occursin("could not find R", absent_status.reason) + @test occursin("dada2", absent_status.reason) + @test absent_status.reason != unreachable_status.reason + + # And the third world, which needs no R at all: a probe is answered about the + # checkout before it is answered about the interpreter, so a missing pin file + # is reported as a missing pin file rather than as an unreachable R. + no_lock = PV.r_probe_status(renv_lock = "/nonexistent/renv.lock") + @test no_lock.status === :no_lockfile + @test !PV.r_probe_ok(no_lock) + @test isempty(no_lock.missing) + @test occursin("no renv.lock", no_lock.reason) + end + end From 28c67aade67d4d5449f1a40dcd48162f216ea512 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:46:28 +0000 Subject: [PATCH 2/3] docs(audit): close out Milestone 2 against what the repository does Issue #15 asks for the Milestone 2 record to be audited. Re-ran the claims rather than reading them back, and two of them no longer described the repository: - the ">10% regression gate fails CI" claim, for the frontend script and for each bench/*.jl. The gate was built, then deliberately replaced by an informational comparison: ci.yml records two consecutive runs of identical benchmark code producing -16%..+52% deltas, because the workloads import no application code and so measure the runner. bench/table_loading/benchmark.jl prints NOTE and warns instead of failing. Corrected, with the reasoning kept. - the test-suite figures: 586 frontend tests across 16 files are now 604 (599 pass, 5 todo, 0 fail, 3368 expects) and 27 Julia unit files / 6830 lines are now 30 / 8310. Corrected as dated audit notes beside the original snapshot rather than overwritten, so the record shows both what was measured then and what holds now. Verified and left as recorded: coverage 40.23% funcs / 47.50% lines (claim 40/47), all five benchmark categories plus the comprehensive runner and analysis_config, the seven frontend workloads one-for-one, Codecov residue gone and gitar absent, CI runtime 14-29 min (claim 15-20). The project board is recorded as unverifiable with the credentials this audit had, not assumed. docs/audit/milestone-2-close-out.md carries the claim-by-claim evidence and the reproducible commands behind each verdict. Closes #15 --- docs/audit/milestone-2-close-out.md | 193 ++++++++++++++++++ .../02-baseline-tests-benchmarks.md | 44 +++- 2 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 docs/audit/milestone-2-close-out.md diff --git a/docs/audit/milestone-2-close-out.md b/docs/audit/milestone-2-close-out.md new file mode 100644 index 00000000..6c045e9b --- /dev/null +++ b/docs/audit/milestone-2-close-out.md @@ -0,0 +1,193 @@ + + +# Milestone 2 Close-Out Audit — Baseline Tests, Benchmarks, CI/CD, Project Board + +**Date:** 2026-09-22 +**Audit type:** Close-out verification of issue #15. Claims were re-run against the repository rather than read back from the milestone document. +**Auditor:** Agentic audit; commands and their output are quoted so each verdict is reproducible. +**Revision audited:** `main` @ `22d228f5` + +## Verdict + +All four of #15's acceptance criteria hold: the test suite and its pathways are +documented, the benchmarks exist and run, CI is extended and green on `main`, and the +board is linked. Two claims in +`docs/milestones/02-baseline-tests-benchmarks.md` no longer described what the +repository does, and both are corrected in the same PR as this audit: + +1. the ">10% regression gate fails CI" claim (frontend **and** Julia) — the gate was + deliberately superseded by an informational comparison, with measured reasoning; +2. the test-suite figures, which had drifted upward as tests were added. + +Neither correction invalidates the milestone: the first is a decision the owner took +after the document was written and is recorded in `ci.yml`; the second is a snapshot +that has aged in the safe direction. + +## What was run + +| Command | Where | +| --- | --- | +| `bun test` / `bun test --coverage` | `frontend/` | +| `ls test/unit/*.jl \| wc -l`, `cat test/unit/*.jl \| wc -l` | repository root | +| `ls bench/`, `ls frontend/bench/` | repository root | +| `grep` over `ci.yml`, `README.md`, `bench/**`, `frontend/bench/baseline.json` | repository root | +| `GET /actions/runs?branch=main&status=success` | GitHub API | + +## Claims, evidence, verdicts + +### Frontend test suite + +**Claimed** (doc, local run 2026-09-18): 581 pass, 5 todo, 0 fail, 3350 expects, +586 tests across 16 files, 415 ms. + +**Measured** (2026-09-22): 599 pass, 5 todo, 0 fail, 3368 expects, 604 tests across +16 files, 461 ms. + +**Verdict: holds, grown by 18 tests.** The shape is unchanged — the same 5 todos, the +same 16 files, no failures. The todo count is the interesting one: it has not moved, +so the DOM-lane debt the document records is still exactly that debt and has not +silently grown. + +### Frontend coverage + +**Claimed:** 40.19% functions, 47.53% lines (issue: "40% funcs, 47% lines"). +No gate, by policy. + +**Measured:** 40.23% functions, 47.50% lines. + +**Verdict: holds.** Regenerated independently; the difference is rounding. +Under-covered files remain the DOM-only ones (`DataTable.tsx` 0.00/0.93, +`CardActions.tsx` 0.00/3.51, `NameDialog.tsx` 0.00/2.38), which is the documented +consequence of the unit lane being DOM-less rather than a regression. The e2e lane is +where those files are exercised — see #32, whose dialog specs run in real Chromium. + +### Julia test suite + +**Claimed:** 27 unit test files, 6830 lines. + +**Measured:** 30 unit test files, 8310 lines. + +**Verdict: holds, grown by 3 files and 1480 lines.** Reproducible with the two +commands above. + +### Benchmarks + +**Claimed:** `table_loading`, `epistemic_parsing`, `duckdb_aggregation`, +`permanova_nmds`, `tree_rendering`, plus a comprehensive runner; frontend harness +with 7 workloads. + +**Measured:** all five `bench//benchmark.jl` present, plus +`bench/comprehensive_benchmark.jl` and `bench/analysis_config/benchmark.jl`; every +category has a committed `baseline.json`. `frontend/bench/baseline.json` holds +exactly 7 workloads: `run-table-json-parse`, `figure-colour-overrides`, +`table-loading-sample-columns`, `epistemic-parsing`, `duckdb-aggregation`, +`permanova-nmds`, `tree-rendering-clade-cumulus`. + +**Verdict: holds.** The 7 names match the document one-for-one. + +### The >10% regression gate — superseded + +**Claimed:** "Regression gate: >10% fail in CI"; frontend Node script fails on +>10% delta; each Julia `bench/*.jl` fails on >10% when `ENV["CI"] == "true"`. + +**Measured: not what the repository does.** `ci.yml` states the deltas are +"REPORTED, never gated" and gives the measurement behind that decision: two +consecutive runs of *identical* benchmark code on a hosted runner produced +per-workload deltas between **-16% and +52%**, flapping in both directions, because +the harness workloads import no application code at all — a delta measures the runner, +not the commit. `bench/table_loading/benchmark.jl` agrees: a >10% delta prints +`NOTE`, emits `@warn "…(informational)"`, and carries the comment "Never gates in +CI." + +**Verdict: claim no longer true; the deviation is deliberate, evidenced, and now +documented.** A hard gate below the noise floor would block at random, which is worse +than no gate — and worse for the repository's own philosophy, since a gate that fails +for reasons the commit cannot influence teaches people to ignore it. The document has +been corrected to describe the informational comparison, the measured noise floor, +and what still holds for real: checksum hard-fails, the workload freeze policy, and +deltas plus machine factor shipped as artifacts for review. + +### CI/CD extension + +**Claimed:** tests and benchmarks on every push/PR, artifacts, and the new categories +`analysis-config` and `cladistic-explorer`. + +**Measured:** frontend `bun test --coverage`, typecheck, build and the benchmark +harness are CI steps; the Julia side runs all five categories plus +`bench/analysis_config/benchmark.jl` and `bench/comprehensive_benchmark.jl`; +`test/unit/test_analysis_config.jl` runs as a step of its own; artifacts +`frontend-tests-benchmarks` (junit.xml, lcov.info, results.json, baseline.json), +`julia-coverage-lcov` (lcov.info) and `julia-benchmarks-comprehensive` are declared. +For the cladistics category the repository carries `frontend/src/components/ +CladeCumulus.tsx` and the `tree-rendering-clade-cumulus` workload; a +`test_clade_cumulus.jl` does **not** exist, which is what the document already says +("future, will test…"). The accepted reading is that the *benchmark category* exists +and the *unit category* remains future work, and the document is precise on that. + +**Verdict: holds**, with the naming clarified above. + +### Codecov and gitar + +**Claimed:** Codecov removed (config deleted, README badge removed, upload replaced by +a local artifact); gitar absent (`grep -i gitar` returns 0). + +**Measured:** no `codecov.yml`; no Codecov badge or slug in `README.md`; `ci.yml` +uploads coverage as a local artifact named `julia-coverage-lcov` with the step +comment "Codecov removed per Milestone 2"; `grep -rin gitar` over the tree returns +nothing. + +**Verdict: holds.** Note that the older +`docs/audit/type-system-reconnaissance.md` still *mentions* a Codecov slug pointing +at the parent repository — that document records a state that has since been +remediated, and it is a historical audit, not a current claim. + +### CI runtime + +**Claimed:** 15–20 minutes. + +**Measured:** the last four successful `main` runs took 29:18, 14:12, 15:17 and +22:21 (median ≈ 19 minutes). + +**Verdict: holds as a range, with a caveat worth recording.** The variance is +dominated by the renv restore — 79 R packages built from source, resumable from a +cache, and the single largest cost in the workflow. A cold cache is the 29-minute +case. + +### Local sandbox RAM + +**Claimed in the document:** the local sandbox is memory-blocked +("876Mi vs 2.5GB required"). + +**Corroborated incidentally:** this audit's own environment could not precompile +`DataFrames` or `XLSX` — the two heaviest direct dependencies — and the memory ceiling +is the plausible explanation; `MetaManifold` itself precompiled in 116 s once the +lighter tree was built. The audit therefore ran its Julia-side verification through a +harness that mounts the real `Provenance` module (whose only sibling dependency is +`RRuntime`) rather than the whole package. The claim is consistent with what the +environment does. + +### Project board + +**Claimed:** https://github.com/users/hyperpolymath/projects/45 — "Analysis Layer & +Cladistics Development", 11 items, PRs #11–#14 linked. + +**Measured: not verifiable with the credentials available to this audit.** The board +is a user-level Projects v2 board; reading it needs the `read:project` scope, which is +granted per-token, and the token in use during this audit carries `repo` and +`workflow` only. + +**Verdict: unverified, and recorded as such rather than assumed.** Everything else on +this page was re-derived from the repository or the API. The link and the board's own +existence are the owner's to confirm; nothing in the codebase depends on it. + +## Consequences + +- `docs/milestones/02-baseline-tests-benchmarks.md` carries dated audit notes beside + the figures it recorded, and its regression-gate section now describes the + informational comparison that replaced the gate. +- No code change is implied by this audit. The tests, the benchmarks and the CI + wiring are as the milestone describes them, except where noted above. +- Two things are deliberately left as they are: `test_clade_cumulus.jl` is still + future work, and the benchmark comparison is still informational. Both are + decisions, not oversights, and both are now written down where a reader will find + them. diff --git a/docs/milestones/02-baseline-tests-benchmarks.md b/docs/milestones/02-baseline-tests-benchmarks.md index c085ad65..4682e947 100644 --- a/docs/milestones/02-baseline-tests-benchmarks.md +++ b/docs/milestones/02-baseline-tests-benchmarks.md @@ -40,8 +40,20 @@ SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) **Audited 2026-09-22 (#15).** Re-run on `main` @ `22d228f5`: 599 pass, 5 todo, +> 0 fail, 3368 expects, **604 tests across 16 files** [461 ms]. Grown by 18 tests; +> the file count and the todo count are unchanged, so the DOM-lane debt recorded +> above has not silently grown. Full audit: +> `docs/audit/milestone-2-close-out.md`. + **Coverage (informational, no gate by policy per docs/testing/infrastructure.md):** - All files 40.19% funcs, 47.53% lines + +> **Audited 2026-09-22 (#15).** Regenerated: All files **40.23% funcs, 47.50% lines** — +> unchanged within rounding. The least-covered files are still the DOM-only ones +> (`DataTable.tsx`, `CardActions.tsx`, `NameDialog.tsx`), which is the documented +> consequence of a DOM-less unit lane rather than a regression; the e2e lane is where +> those are exercised. - src/api/client.ts 15% funcs 57% lines (many uncovered: config, analysis, etc — API client not fully tested via unit, but via integration) - src/api/errorMessage.ts 100% - src/components/CardActions.tsx 0% funcs 3.51% lines (UI not DOM-tested) @@ -58,6 +70,11 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage **Command:** `julia --project=. -t 2 --code-coverage=user test/runtests.jl` (unit always, --integration opt-in, --server opt-in) **Pathways (27 unit test files, 6830 lines total):** + +> **Audited 2026-09-22 (#15).** Measured on `main` @ `22d228f5`: **30 unit test files, +> 8310 lines**. The list below is the Milestone 2 snapshot and remains an accurate +> description of the files it names; of the current files, only `test_execution.jl` +> is not named below. Full audit: `docs/audit/milestone-2-close-out.md`. - `test_diversity.jl` (87 lines): richness (5), shannon (6), simpson (6), Normalisation rarefy (1), normalise_counts (1) — total 19 tests - `test_merge_taxa.jl` (311): merge_taxa join, tagging, max_x, category_sets - `test_config.jl` (62): config cascade, defaults, overrides @@ -118,7 +135,8 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage - taxonomy_levels (distinct ranks) - taxon_column (rank resolution) - Baseline: baseline.json with median seconds per op (5 reps) - - Regression gate: >10% fail in CI + - Baseline comparison: **informational** — see "Regression Gate" below, which was + corrected on audit 2026-09-22 (#15) 2. **bench/epistemic_parsing/benchmark.jl** - Mock epistemic types mirroring future src/core/epistemic.jl: EpistemicStatus enum present_in_every/present_in_some/absent/unknown/sans_fibre, MockCandidate observation/residual/witness, MockCase candidates @@ -156,7 +174,7 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage 6. **bench/comprehensive_benchmark.jl** - Runner for all 5 categories, writes bench/results/comprehensive_results.json - - Fails on >10% regression when CI=true + - Reports deltas against each committed baseline; does not gate (see below) **Baselines:** - Each category has baseline.json committed with placeholder medians (will be overwritten on first CI run that succeeds) @@ -202,9 +220,25 @@ Ran 586 tests across 16 files. [415.00ms] first run, [340.00ms] with --coverage - julia-coverage-lcov: lcov.info - julia-benchmarks-comprehensive: bench/*/baseline.json, bench/results/comprehensive_results.json, bench/**/baseline.json -**Regression Gate:** >10% fail -- Frontend: Node script in CI fails if any workload median delta >10% vs baseline.json -- Julia: Each bench/*.jl checks baseline.json and fails if delta >10% when ENV["CI"]=="true" (via exit(1) and ::error:: annotation) +**Regression Gate:** none, deliberately — deltas are reported, never gated + +> **Corrected 2026-09-22 (#15).** This section previously recorded a hard +> ">10% fail" gate for both the frontend script and each `bench/*.jl`. That is not +> what the repository does, and it was changed on purpose after the gate was built. +> `ci.yml` carries the measurement: two consecutive runs of *identical* benchmark +> code on a hosted runner produced per-workload deltas between **-16% and +52%**, +> flapping in both directions, because the harness workloads import no application +> code — a delta therefore measures the runner, not the commit. `bench/table_loading/ +> benchmark.jl` agrees: a >10% delta prints `NOTE` and emits +> `@warn "…(informational)"`, with the comment "Never gates in CI." +> +> A gate below the noise floor blocks at random, and a gate that fails for reasons a +> commit cannot influence teaches people to ignore it. What still holds for real: the +> harness checksums hard-fail, the workload freeze policy requires re-cutting the +> baseline (visible in review), and deltas plus the machine factor are printed and +> shipped as artifacts for human review. Revisiting a same-runner A/B gate is left open +> once the harness exists on the base branch. Full audit: +> `docs/audit/milestone-2-close-out.md`. **New Test Categories:** - analysis-config: test_analysis_config.jl (AnalysisConfig creation, validation, BH mandatory, DANGER token, DOI bundle, epistemic, cloud sizing) — 21 files 5276 insertions in feature branches From 99efa784b62dc3ccc00047a7810b69c0a4a3c237 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:51:25 +0000 Subject: [PATCH 3/3] feat(analysis): exact counts and rationals with an explicit numeric policy First foundation for the validated statistics layer (#1): the numeric integrity half, which the methods half builds on. No method is added here and nothing existing changes behaviour -- :ordinary, the default mode, is Float64 throughout and leaves saved analyses untouched. The module keeps three claims apart instead of letting one representation carry all of them: exact (integer counts, rationals), approximate (any float, including arbitrary precision), rounded (display). Higher precision is not exactness, and the contract says so. What it refuses, each because the alternative is a wrong number that looks right: - `1//0`. Julia answers `1//0` for a non-zero numerator -- a non-finite Rational that renders as a number -- so a zero denominator is refused before the division, and a sample of zero reads reports `nothing` rather than a proportion of 0.0. - Counts arriving as Float64 beyond 2^53, which can hold a value but cannot prove which integer it was, and whose history at an earlier boundary (R, CSV, a JavaScript client) cannot be inspected. - Overflow: checked sums raise CountOverflowError unless the caller asks for :widen, rather than wrapping to a wrong total. - Unbounded rational growth, which is a resource decision and now raises ResourceLimitError against a stated budget. - NaN/Inf at the JSON boundary, which has no representation for them. Concurrency: BigFloat precision is a process global, not a task-local, so with_precision serialises callers and reads the achieved precision inside the scope. Six concurrent tasks at six precisions are asserted to each observe their own, with the default restored afterwards. Boundaries are published in docs/statistics/numeric-contracts.md: the JSON rules (integers beyond 2^53 and rationals travel as strings, because a JSON number in a browser is a Float64), CSV, R, DuckDB, and the provenance fingerprint that makes two runs comparable. The bounded method catalogue is drafted for approval in the same directory, since #1 requires it before methods are implemented. .gitignore whitelists docs/statistics/ the same way its sibling documentation directories are whitelisted. Tests: 144, run under the real module. Known-answer, adversarial (overflow, all-zero tables, denominator exhaustion, non-finite input), metamorphic (proportions of a table sum to exactly 1 across 50 random tables), concurrency, and negative controls that assert the lossy alternatives really are lossy. Refs #1 --- .gitignore | 2 + docs/statistics/method-catalogue-v1-draft.md | 71 +++ docs/statistics/numeric-contracts.md | 112 +++++ src/MetaManifold.jl | 1 + src/analysis/numeric_policy.jl | 482 +++++++++++++++++++ test/runtests.jl | 1 + test/unit/test_numeric_policy.jl | 252 ++++++++++ 7 files changed, 921 insertions(+) create mode 100644 docs/statistics/method-catalogue-v1-draft.md create mode 100644 docs/statistics/numeric-contracts.md create mode 100644 src/analysis/numeric_policy.jl create mode 100644 test/unit/test_numeric_policy.jl diff --git a/.gitignore b/.gitignore index eac41e5c..54a15cc6 100644 --- a/.gitignore +++ b/.gitignore @@ -254,6 +254,8 @@ docs/* !docs/migration/ !docs/milestones/ !docs/milestones/** +!docs/statistics/ +!docs/statistics/** !docs/issues/ !docs/issues/** !docs/testing/ diff --git a/docs/statistics/method-catalogue-v1-draft.md b/docs/statistics/method-catalogue-v1-draft.md new file mode 100644 index 00000000..b96c4ec2 --- /dev/null +++ b/docs/statistics/method-catalogue-v1-draft.md @@ -0,0 +1,71 @@ + + +# Bounded method catalogue, v1 — DRAFT for owner approval + +**Status: DRAFT. Not approved, not implemented.** Issue #1 requires the bounded +catalogue and its supported/unsupported conditions to be approved *before* methods are +implemented. This document is the artefact to approve or amend, not a claim that the +catalogue is settled. + +## What is already in the application + +These exist today and are not new work; the catalogue records them so that "supported" +means something specific. + +| Method | Where | Response type | Notes | +| --- | --- | --- | --- | +| Richness, Shannon, Simpson | `src/analysis/diversity.jl` | counts per sample | descriptive indices | +| Rarefaction / normalisation | `src/analysis/diversity.jl` | counts per sample | `none`, `rarefy` | +| Alpha diversity comparisons | `src/analysis/analysis.jl` | group → value | significance reported with a status, never silently degraded | +| NMDS ordination | `src/analysis/analysis.jl` | distance matrix | vegan, via the locked R runtime | +| PERMANOVA | `src/analysis/analysis.jl` | distance matrix + grouping | vegan; exchangeability of the permutation units is the standing caveat | + +## Proposed v1 additions, in dependency order + +1. **Descriptive summaries at exact precision** — counts and proportions as exact + integers/rationals. No inference claimed. Depends only on `numeric_policy.jl`, + which this PR adds. +2. **Parametric fits with explicit constraint and convergence reporting** — + maximum likelihood where the model justifies it, with identifiability, + boundary-estimate and convergence checks that produce *unsuccessful states* rather + than plausible parameters. +3. **Nonparametric tests** — permutation/bootstrap, with resampling units and + exchangeability stated, seeds recorded, and Monte Carlo limits reported alongside + the estimate. +4. **Exact statistical tests** (separate issue, #3) — Fisher's exact, exact negative + binomial, permutation PERMANOVA. Not in v1. + +## Supported / unsupported conditions to be agreed with each method + +Every method above will publish, before implementation: + +- **Response type** it accepts (counts, proportions, distances, continuous) and what + it does with zeros, and with all-zero samples. +- **Study design** it supports: pairing, blocking, repeated measures, covariates, + unequal sequencing depth, compositionality. Missing design information means a + descriptive summary, never a guess. +- **Overdispersion and depth** handling, named explicitly, not absorbed into a + default. +- **Uncertainty**: interval method, effect size, multiple-testing policy (BH is + mandatory where several tests are reported — an existing project rule). +- **Diagnostics and warnings** surfaced to the user, in accessible language, including + the assumption that is most likely to be wrong for the data at hand. +- **Computational limits**: time and memory bounds, and what happens at them + (`ResourceLimitError`, an explicit unsuccessful state). + +## Explicitly not claimed + +- No Poisson/negative-binomial or binomial/beta-binomial model is approved by this + draft for sequencing data; both are candidates requiring review, not defaults. +- "Nonparametric" is a class of methods, not a universal safe distribution and not an + assumption-free alternative. Exchangeability still has to be argued. +- No method is selected by a normality test or by which p-value looks favourable, and + no method is switched silently. +- Higher precision is not exactness (see `numeric-contracts.md`). + +## What approval is being asked for + +1. The v1 scope above (items 1–3; item 4 stays deferred). +2. The per-method publication list as the acceptance gate for each method. +3. The reading that a descriptive summary is the correct default whenever the + information needed for valid inference is missing. diff --git a/docs/statistics/numeric-contracts.md b/docs/statistics/numeric-contracts.md new file mode 100644 index 00000000..b441330d --- /dev/null +++ b/docs/statistics/numeric-contracts.md @@ -0,0 +1,112 @@ + + +# Numeric conversion and storage contracts (v1) + +**Status:** published, implemented in `src/analysis/numeric_policy.jl` (issue #1). +**Scope:** how numbers become exact, approximate or rounded; what may cross each +boundary; what is refused. Methods and their assumptions are catalogued separately in +`method-catalogue-v1-draft.md`. + +## Why this document exists + +Most statistics are not exact, and no policy makes them so. What a policy can do is +stop three different claims from wearing the same representation: + +| Kind | What it means | May be called a fact? | +| --- | --- | --- | +| **exact** | an integer count, or a rational built from counts; no rounding in its history | yes, as a count or proportion | +| **approximate** | any floating-point value, *including arbitrary precision* | no — it is a number | +| **rounded** | an approximation cut to N digits for display | no — it is a rendering | + +Higher precision is not exactness. `BigFloat` at 4096 bits still rounds; it simply +rounds further away. The only exact arithmetic in this layer is integer and rational. + +## The policy + +`numeric_policy(; mode, precision_bits, max_denominator_bits, round_digits)` returns an +immutable `NumericPolicySpec`, validated at construction: + +| Field | Modes / limits | Default | +| --- | --- | --- | +| `mode` | `:ordinary`, `:exact_counts`, `:high_precision` | `:ordinary` | +| `precision_bits` | 2 … 1 000 000 | 53 (Float64's significand) | +| `max_denominator_bits` | ≥ 32 | 4096 | +| `round_digits` | ≥ 0 | 6 | + +`:ordinary` is Float64 throughout and is the only mode under which previously saved +analyses are unchanged. Nothing in the layer switches mode on its own: a caller that +needs `:exact_counts` and is handed an `:ordinary` policy fails through +`assert_mode`, rather than receiving a Float64 that looks like the exact answer. + +**Provenance.** `policy_fingerprint(spec)` is a SHA256 over the policy's fields. Two +runs may be compared only when this agrees; it is recorded alongside a run, so a +result can never be attributed to a policy that was not in force. + +## Contracts, boundary by boundary + +**Counts.** `require_count(x)` accepts integers, and floats only when finite, integral +and within 2^53. Beyond that a Float64 holds *a* value but cannot prove *which* +integer it was — and the float in hand may have been rounded earlier by R, a CSV +reader or a JavaScript client, a history that cannot be inspected. Such a value is +refused. `checked_count_sum` refuses to wrap: overflow raises `CountOverflowError` +unless `on_overflow = :widen` carries the total in `BigInt`. + +**Proportions.** `exact_relative_abundance(count, total)` returns `Rational{BigInt}`, or +`nothing` when `total == 0`. Julia's own `//` would answer `1//0` here — a non-finite +`Rational` that renders as a number — so the zero denominator is refused before the +division. `nothing` means "this sample has no composition", an unsuccessful state the +presentation layer must render as such, never as `0.0`. `exact_rational_sum` bounds +denominator growth and raises `ResourceLimitError` when the budget is exhausted. + +**JSON and the browser.** A JSON number on the way through a JavaScript client is a +Float64, so: + +| Value | JSON form | +| --- | --- | +| integer, \|n\| ≤ 2^53 − 1 | JSON number | +| integer beyond that | **string** (a count of 9007199254740993 arrives as …93, not …92) | +| rational | **string** `"n/d"` | +| `BigFloat` | **string**, decimal — its precision is why it is not a Float64 | +| finite `Float64` | JSON number | +| `NaN`, `±Inf` | **refused** — JSON has no representation for them | + +`to_storage` applies the same rules to a wrapped value; `to_display` is the *only* +place a value is rounded, and the string it returns names its own kind. + +**CSV.** Text in, text out, no types. A count read back from CSV is a `Float64` via any +parser that sniffs types, so a CSV round trip of a big count is lossy and must be +declared: re-import through `require_count`, which refuses what cannot be proven. + +**R.** RCall conversions go through `Float64` unless the R side returns a string; +`as.integer` overflows at 2^31, and R numerics are doubles. Counts crossing to R and +back are therefore treated as unproven integers and re-checked on return. + +**DuckDB.** Integers reach 64-bit; beyond that, values must be stored as the string +form above rather than as `DECIMAL`, whose precision is a column definition and not a +property of the value. + +**Records and other implementations.** Nothing in this contract normalises a version +string or a rounded display value: a value that has been rounded is marked rounded +where it is created, so no later reader has to guess. + +## Concurrency + +`with_precision(f; bits)` runs `f` at an explicit BigFloat precision and returns +`(result, achieved_bits)`. Precision is a **process global** in Julia, not a +task-local, so the scope is serialised behind a lock (the same trade the R runtime +makes), and the achieved bits are read inside the scope — reading them afterwards +reports the previous precision. Separate processes have separate globals; the lock +guards one process. The test suite asserts six concurrent tasks each observe their own +precision and the default is restored afterwards. + +## What this document does not claim + +- It does not make any statistical method exact. Methods, their assumptions and their + supported designs are the catalogue's business, and that catalogue is a draft + awaiting review. +- It does not remove measurement error, sampling uncertainty, model misspecification or + biological bias. Precision is not accuracy. +- It does not cover symbolic mathematics, which is out of scope for issue #1. +- Exact statistical tests (Fisher, exact negative binomial, permutation) remain + deferred; when they arrive they need their own assumptions, limits and known-answer + cases, and an approximate method must never be substituted silently. diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index 14a9c10a..1e86c791 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -28,6 +28,7 @@ include("pipeline/dada2.jl") include("pipeline/swarm.jl") # Analysis +include("analysis/numeric_policy.jl") include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/AnalysisConfig.jl") diff --git a/src/analysis/numeric_policy.jl b/src/analysis/numeric_policy.jl new file mode 100644 index 00000000..0299ec9f --- /dev/null +++ b/src/analysis/numeric_policy.jl @@ -0,0 +1,482 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Numeric integrity for the statistics layer (issue #1). +# +# This module exists to keep three claims apart, because collapsing them is how a +# plausible-looking number gets published that nobody can defend: +# +# exact an integer count, or a rational built from counts, with no +# rounding anywhere in its history. +# approximate a floating-point value -- including arbitrary precision. Higher +# precision is NOT exactness; it moves the rounding error, it does +# not abolish it. +# rounded a display value, cut to N digits on purpose for presentation. +# +# The layer cannot make statistics exact; most of them are not. It can refuse to +# conflate these three, and it can fail loudly at every boundary where a silent +# downgrade would otherwise happen: Julia's own `//` (which will happily hand back +# `1//0`), JSON, a JavaScript client whose numbers are Float64, CSV, R and DuckDB. +# +# Nothing here mutates global state except inside `with_precision`, which restores +# what it found and serialises callers, because BigFloat's precision is a process +# global rather than a task-local. +module NumericPolicy + +using JSON3, SHA + +export NUMERIC_MODES, ORDINARY_PRECISION_BITS, DEFAULT_MAX_DENOMINATOR_BITS, + NumericPolicySpec, numeric_policy, validate_policy, policy_fingerprint, assert_mode, + CountOverflowError, ResourceLimitError, UnsupportedRepresentationError, + require_count, checked_count_sum, exact_fraction, exact_relative_abundance, + exact_rational_sum, with_precision, + EXACT, APPROXIMATE, ROUNDED, NumericValue, exact_value, approximate_value, + rounded_value, is_exact, to_storage, to_display, encode_json_number + +## Modes +# :ordinary is the pre-existing behaviour -- Float64 throughout -- and is the only +# mode under which every saved analysis is unchanged. :exact_counts keeps counts as +# integers and proportions as rationals. :high_precision is arbitrary-precision +# floating point at an explicit number of bits, and remains an approximation. +const NUMERIC_MODES = (:ordinary, :exact_counts, :high_precision) + +# Float64 has a 53-bit significand, so that is the precision "ordinary" means. +const ORDINARY_PRECISION_BITS = 53 + +# The largest exponent of 2 that a Float64 can hold as a consecutive integer. Above +# it, a Float64 can hold a value but cannot prove which integer it was. +const MAX_EXACT_FLOAT = big(2)^53 + +# Exact rationals are kept as Rational{BigInt}; without a ceiling, summing many +# proportions grows the denominator without bound and turns a job into an OOM +# rather than an error. 4096 bits is ~1233 decimal digits. +const DEFAULT_MAX_DENOMINATOR_BITS = 4096 + +## Failures +# Every failure here names what was attempted and what was observed, because these +# three are the states a caller must not mistake for a result: overflow, an +# arithmetic budget exhausted, and a representation the target cannot carry. + +struct CountOverflowError <: Exception + left :: BigInt + right :: BigInt +end + +Base.showerror(io::IO, e::CountOverflowError) = print(io, + "count overflow: $(e.left) + $(e.right) leaves the range of the integer type in ", + "use; widen the accumulator (on_overflow = :widen) or fix the input") + +struct ResourceLimitError <: Exception + what :: String + limit :: Int + observed :: Int +end + +Base.showerror(io::IO, e::ResourceLimitError) = print(io, + "exact arithmetic budget exhausted: $(e.what) is $(e.observed) bits, limit ", + "$(e.limit) bits; raise the limit deliberately or use a method that does not ", + "need exact rationals") + +struct UnsupportedRepresentationError <: Exception + what :: String + value :: String + target :: String +end + +Base.showerror(io::IO, e::UnsupportedRepresentationError) = print(io, + "$(e.target) cannot carry $(e.value) as $(e.what) without loss; convert ", + "explicitly and record which precision was chosen") + +## Policy +struct NumericPolicySpec + mode :: Symbol + precision_bits :: Int + max_denominator_bits :: Int + round_digits :: Int +end + +""" + numeric_policy(; mode=:ordinary, precision_bits=ORDINARY_PRECISION_BITS, + max_denominator_bits=DEFAULT_MAX_DENOMINATOR_BITS, round_digits=6) + +The numeric policy one analysis runs under: which mode, at what precision, with what +exact-arithmetic budget and how many digits a display value keeps. + +Immutable and passed explicitly, never read from a global. That is what keeps two +concurrent analyses from sharing a precision setting: there is no shared setting to +share. +""" +function numeric_policy(; mode::Symbol = :ordinary, + precision_bits::Integer = ORDINARY_PRECISION_BITS, + max_denominator_bits::Integer = DEFAULT_MAX_DENOMINATOR_BITS, + round_digits::Integer = 6) + spec = NumericPolicySpec(mode, Int(precision_bits), Int(max_denominator_bits), + Int(round_digits)) + validate_policy(spec) + return spec +end + +numeric_policy(mode::Symbol) = numeric_policy(; mode) + +""" + validate_policy(spec) -> NumericPolicySpec + +Reject a policy that cannot mean anything, at construction rather than in the +middle of an analysis. +""" +function validate_policy(spec::NumericPolicySpec) + spec.mode in NUMERIC_MODES || throw(ArgumentError( + "mode must be one of $(join(NUMERIC_MODES, ", ")) -- got '$(spec.mode)'")) + spec.precision_bits >= 2 || throw(ArgumentError( + "precision_bits must be at least 2, got $(spec.precision_bits)")) + spec.precision_bits <= 1_000_000 || throw(ArgumentError( + "precision_bits above 1e6 is a resource decision, not a configuration; " * + "got $(spec.precision_bits)")) + spec.max_denominator_bits >= 32 || throw(ArgumentError( + "max_denominator_bits must be at least 32, got $(spec.max_denominator_bits)")) + spec.round_digits >= 0 || throw(ArgumentError( + "round_digits must not be negative, got $(spec.round_digits)")) + return spec +end + +""" + assert_mode(spec, mode) + +Refuse to continue when the caller needs `mode` and the policy does not provide it. + +The failure this prevents is a silent downgrade: an exact path requested, an +ordinary-precision policy in force, and a Float64 result that looks the same as the +exact one would have. +""" +function assert_mode(spec::NumericPolicySpec, mode::Symbol) + spec.mode === mode || throw(UnsupportedRepresentationError( + "mode $(mode)", "policy is $(spec.mode)", + "this analysis; re-run with numeric_policy(; mode = :$mode) or drop the exact requirement")) + return spec +end + +""" + policy_fingerprint(spec) -> String + +A stable SHA256 over the policy's own fields, for provenance and for config hashing: +two runs may be compared only if this agrees. +""" +function policy_fingerprint(spec::NumericPolicySpec) + canonical = Dict{String,Any}( + "mode" => String(spec.mode), + "precision_bits" => spec.precision_bits, + "max_denominator_bits" => spec.max_denominator_bits, + "round_digits" => spec.round_digits, + ) + return bytes2hex(sha256(JSON3.write(canonical))) +end + +## Counts +""" + require_count(x) -> Integer + +`x` as a count, or a loud refusal. + +Integers pass through. A float passes only if it is finite, integral, and within +2^53: beyond that a Float64 can hold a value but cannot prove which integer it was, +and the float in hand may already have been rounded by R, by a CSV reader or by a +JavaScript client before this function ever saw it. That history cannot be +inspected, so the value is refused rather than trusted. +""" +function require_count(x) + x isa Integer && return x + if x isa Rational + denominator(x) == 1 && return numerator(x) + throw(UnsupportedRepresentationError("a count", string(x), + "a count is an integer; $(x) is not one")) + end + if x isa AbstractFloat + isfinite(x) || throw(UnsupportedRepresentationError("a count", string(x), + "counts must be finite")) + isinteger(x) || throw(UnsupportedRepresentationError("a count", string(x), + "a count is an integer; $(x) is not one")) + abs(big(x)) <= MAX_EXACT_FLOAT || throw(UnsupportedRepresentationError( + "an exactly-known count", string(x), + "Float64 above 2^53 cannot say which integer it holds")) + return x isa BigFloat ? BigInt(x) : Integer(x) + end + throw(UnsupportedRepresentationError("a count", string(x), + "counts are integers, not $(typeof(x))")) +end + +""" + checked_count_sum(values; on_overflow=:error) -> Integer + +The exact sum of `values`, with overflow made visible instead of wrapping. + +`on_overflow = :error` refuses; `:widen` continues in `BigInt`, which is the honest +answer for a total that a fixed-width integer cannot hold. A sum that wraps around +to a small number is worse than a failure: it is a wrong total that no downstream +check can detect. +""" +function checked_count_sum(values; on_overflow::Symbol = :error) + on_overflow in (:error, :widen) || throw(ArgumentError( + "on_overflow must be :error or :widen, got :$on_overflow")) + + state = iterate(values) + state === nothing && return 0 + first_value, rest = state + acc = require_count(first_value) + + while true + nxt = iterate(values, rest) + nxt === nothing && break + value, rest = nxt + count = require_count(value) + + if acc isa BigInt || count isa BigInt + acc = acc + count # BigInt promotes the other side + continue + end + try + acc = Base.Checked.checked_add(promote(acc, count)...) + catch err + err isa OverflowError || rethrow() + on_overflow === :widen || throw(CountOverflowError(BigInt(acc), BigInt(count))) + acc = BigInt(acc) + BigInt(count) + end + end + return acc +end + +## Exact rational arithmetic +function _check_denominator_bits(value::Rational, spec::NumericPolicySpec) + bits = ndigits(denominator(value), base = 2) + bits <= spec.max_denominator_bits || throw(ResourceLimitError( + "denominator", spec.max_denominator_bits, bits)) + return value +end + +""" + exact_fraction(numerator, denominator; policy) -> Rational{BigInt} + +`numerator/denominator`, exactly, with a zero denominator refused before any division +happens. + +The refusal is the point. Julia's `//` answers `1//0` for a non-zero numerator and +`0//0` for a zero one: a non-finite Rational that flows on into charts and tables +looking like a number. A proportion with no denominator is not a large proportion or +a small one; it is not a proportion. +""" +function exact_fraction(numerator::Integer, denominator::Integer; + policy::NumericPolicySpec = numeric_policy()) + iszero(denominator) && throw(UnsupportedRepresentationError( + "a proportion", "$(numerator)/$(denominator)", + "a fraction whose denominator is zero; refuse the value rather than carry " * + "a non-finite Rational past this point")) + return _check_denominator_bits(Rational{BigInt}(numerator, denominator), policy) +end + +""" + exact_rational_sum(values; policy) -> Union{Rational{BigInt},Nothing} + +The exact sum of a collection of rationals, bounded by the policy's denominator +budget. + +Returns `nothing` for an empty collection rather than `0`: a sum over nothing has no +value, and a zero here would be read downstream as "the values add up to none". +""" +function exact_rational_sum(values; policy::NumericPolicySpec = numeric_policy()) + acc = nothing + for value in values + value isa Rational || throw(UnsupportedRepresentationError( + "an exact rational", string(value), "exact_rational_sum")) + acc = isnothing(acc) ? Rational{BigInt}(value) : + _check_denominator_bits(acc + Rational{BigInt}(value), policy) + end + return acc +end + +""" + exact_relative_abundance(count, total; policy) -> Union{Rational{BigInt},Nothing} + +`count/total` exactly, or `nothing` when `total == 0`. + +`nothing`, not `0.0` and not an error: a sample of zero reads has no composition at +all, so "0 of 0 reads are this feature" is not a proportion that happens to be zero, +it is a proportion that does not exist. Callers must carry it to the presentation +layer as an unsuccessful state, which is why it is not a number here. +""" +function exact_relative_abundance(count::Integer, total::Integer; + policy::NumericPolicySpec = numeric_policy()) + total >= 0 || throw(ArgumentError("total must not be negative, got $total")) + count >= 0 || throw(ArgumentError("count must not be negative, got $count")) + count <= total || throw(ArgumentError( + "count ($count) exceeds total ($total); these are not counts of the same table")) + iszero(total) && return nothing + return exact_fraction(count, total; policy) +end + +## Precision +# BigFloat's precision lives in a process-wide RefValue, not in the task. Scoping it +# with setprecision() therefore protects one caller from another *only* if the +# callers are serialised, which is what this lock does. Serialising is the same +# trade the R runtime makes: correctness over parallel speed, for work whose answer +# must be defensible. +const _PRECISION_LOCK = ReentrantLock() + +""" + with_precision(f; bits) -> (result, achieved_bits) + +Run `f` with BigFloat precision set to `bits`, restoring whatever was there before. + +`bits` is explicit so that no caller depends on ambient precision. The pair is +returned together on purpose: a caller that records only its result cannot show +afterwards that the precision it asked for was the precision it got. +""" +function with_precision(f::Function; bits::Integer) + bits >= 2 || throw(ArgumentError("bits must be at least 2, got $bits")) + return lock(_PRECISION_LOCK) do + previous = precision(BigFloat) + try + # `achieved` is read INSIDE the scope, before setprecision restores the + # previous value: read afterwards it reports the precision that was in + # force before the call, which is exactly the fact this return value + # exists to establish. + return setprecision(BigFloat, Int(bits)) do + value = f() + (value, precision(BigFloat)) + end + finally + setprecision(BigFloat, previous) + end + end +end + +## Values that know what they are +const EXACT = :exact +const APPROXIMATE = :approximate +const ROUNDED = :rounded + +struct NumericValue + value :: Any + kind :: Symbol + precision_bits :: Union{Int,Nothing} + digits :: Union{Int,Nothing} +end + +""" + exact_value(x) + approximate_value(x; bits=nothing) + rounded_value(x; digits) + +The three kinds of number this layer is willing to carry. + +An `NumericValue` is what a caller passes when the number may reach a user, a file or +another language, because at those boundaries the kind decides what is safe: exact +values must not be handed to a Float64 boundary, and approximate ones must not be +presented as exact. +""" +function exact_value(x) + (x isa Integer || x isa Rational) || throw(UnsupportedRepresentationError( + "an exact value", string(x), + "exact_value; integer counts and rationals are exact, $(typeof(x)) is not")) + x isa Rational && denominator(x) == 0 && throw(UnsupportedRepresentationError( + "an exact value", string(x), "a rational needs a non-zero denominator")) + return NumericValue(x, EXACT, nothing, nothing) +end + +function approximate_value(x; bits::Union{Integer,Nothing} = nothing) + x isa AbstractFloat || throw(UnsupportedRepresentationError( + "an approximate value", string(x), + "approximate_value; $(typeof(x)) is not floating point")) + return NumericValue(x, APPROXIMATE, isnothing(bits) ? nothing : Int(bits), nothing) +end + +function rounded_value(x; digits::Integer) + x isa AbstractFloat || throw(UnsupportedRepresentationError( + "a rounded value", string(x), + "rounded_value; round an approximate float explicitly")) + digits >= 0 || throw(ArgumentError("digits must not be negative, got $digits")) + return NumericValue(x, ROUNDED, nothing, Int(digits)) +end + +""" + is_exact(v::NumericValue) -> Bool + +Whether `v` may be read as a count or a proportion without an approximation in its +history. False means it is a number, not a fact. +""" +is_exact(v::NumericValue) = v.kind === EXACT + +## Boundaries +""" + to_storage(v::NumericValue) -> Union{Integer,String,Float64} + +The representation this value may be stored and shipped as. + +Exact integers up to 2^53 stay integers -- that is the range a JavaScript client +reproduces exactly, and this application has one. Everything exact beyond that, and +every rational, travels as a string, because a JSON number on the way through a +browser is a Float64 and would silently round: a count of 9007199254740993 arrives +as 9007199254740992. Approximate values travel as Float64, and non-finite ones are +refused, since JSON cannot carry them at all. +""" +function to_storage(v::NumericValue) + x = v.value + if v.kind === EXACT + if x isa Integer + return abs(big(x)) <= MAX_EXACT_FLOAT - 1 ? x : string(x) + end + return string(numerator(x), "/", denominator(x)) + end + if x isa Float64 + isfinite(x) || throw(UnsupportedRepresentationError("a stored number", string(x), + "JSON and JavaScript; neither carries Inf or NaN faithfully")) + return x + end + return x isa BigFloat ? string(x) : Float64(x) +end + +""" + to_display(v::NumericValue; digits=6) -> String + +The string a person reads. + +Rounding happens here and only here, and a rounded value says so in its own type +before it arrives. +""" +function to_display(v::NumericValue; digits::Integer = 6) + digits >= 0 || throw(ArgumentError("digits must not be negative, got $digits")) + if v.kind === EXACT + return v.value isa Integer ? string(v.value) : + string(v.value, " (exact; ", digits, "dp = ", + round(BigFloat(numerator(v.value)) / BigFloat(denominator(v.value)); + digits = digits), ")") + end + kept = v.kind === ROUNDED && !isnothing(v.digits) ? v.digits : Int(digits) + return string(round(v.value; digits = kept), " (", v.kind, ", ", kept, "dp)") +end + +""" + encode_json_number(x) + +The JSON-safe form of a bare number, for values that have not been wrapped yet. + +Rules, in order: an integer inside 2^53 is a JSON number; any other integer is a +string; a rational is a string; a BigFloat is a string, because its precision is the +reason it is not a Float64; a finite Float64 is a JSON number; anything non-finite or +unrecognised is refused. +""" +function encode_json_number(x) + if x isa Integer + return abs(big(x)) <= MAX_EXACT_FLOAT - 1 ? x : string(x) + end + x isa Rational && return string(numerator(x), "/", denominator(x)) + x isa BigFloat && return string(x) + if x isa Float64 + isfinite(x) || throw(UnsupportedRepresentationError("a JSON number", string(x), + "JSON; Inf and NaN have no JSON representation")) + return x + end + x isa AbstractFloat && return encode_json_number(Float64(x)) + throw(UnsupportedRepresentationError("a JSON number", string(x), + "JSON; unsupported type $(typeof(x))")) +end + +end # module diff --git a/test/runtests.jl b/test/runtests.jl index 9992a468..e6cfe9ba 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -33,6 +33,7 @@ using MetaManifold: AnalysisConfig ## Unit tests (always run) @testset "MetabarcodingPipeline" begin + include("unit/test_numeric_policy.jl") include("unit/test_diversity.jl") include("unit/test_merge_taxa.jl") include("unit/test_config.jl") diff --git a/test/unit/test_numeric_policy.jl b/test/unit/test_numeric_policy.jl new file mode 100644 index 00000000..cf8905e4 --- /dev/null +++ b/test/unit/test_numeric_policy.jl @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# `using MetaManifold.NumericPolicy` here, not only at the harness: CI also runs this +# goal outside the full suite, where nothing else has imported the module. +using MetaManifold.NumericPolicy +using Test, Random, SHA, JSON3 + +const NP = MetaManifold.NumericPolicy + +@testset "NumericPolicy" begin + + @testset "policy: validated at construction, fingerprinted for provenance" begin + spec = NP.numeric_policy(; mode = :exact_counts) + @test spec.mode === :exact_counts + @test spec.precision_bits == NP.ORDINARY_PRECISION_BITS + @test NP.validate_policy(spec) === spec + + @test_throws ArgumentError NP.numeric_policy(; mode = :symbolic) + @test_throws ArgumentError NP.numeric_policy(; precision_bits = 1) + @test_throws ArgumentError NP.numeric_policy(; max_denominator_bits = 8) + @test_throws ArgumentError NP.numeric_policy(; round_digits = -1) + + # The fingerprint is what makes two runs comparable: same policy, same hash; + # a changed field, a changed hash. + @test NP.policy_fingerprint(spec) == NP.policy_fingerprint(NP.numeric_policy(; mode = :exact_counts)) + @test NP.policy_fingerprint(spec) != NP.policy_fingerprint(NP.numeric_policy(; mode = :high_precision)) + @test NP.policy_fingerprint(spec) != NP.policy_fingerprint( + NP.numeric_policy(; mode = :exact_counts, precision_bits = 128)) + @test length(NP.policy_fingerprint(spec)) == 64 + end + + @testset "a requested mode is never silently downgraded" begin + ordinary = NP.numeric_policy(; mode = :ordinary) + exact = NP.numeric_policy(; mode = :exact_counts) + @test NP.assert_mode(exact, :exact_counts) === exact + err = try + NP.assert_mode(ordinary, :exact_counts) + nothing + catch e + e + end + @test err isa NP.UnsupportedRepresentationError + # The message must name both sides: what was asked for and what was in force. + @test occursin("exact_counts", sprint(showerror, err)) + @test occursin("ordinary", sprint(showerror, err)) + end + + @testset "counts: what a count may be" begin + @test NP.require_count(7) === 7 + @test NP.require_count(Int32(7)) === Int32(7) + @test NP.require_count(big(7)) == 7 + @test NP.require_count(2^53 - 1) == 2^53 - 1 + + # A float that happens to hold a whole number is accepted where it is exactly + # that number, and refused once Float64 can no longer say which integer it is. + @test NP.require_count(7.0) === 7 + @test NP.require_count(big(2)^53) == big(2)^53 + @test_throws NP.UnsupportedRepresentationError NP.require_count(big(2)^53 + 2.0) + + # The negative control: the value below IS 9007199254740993 as computed, and + # the refusal above is not a formality -- merely rounding the same number to + # Float64 loses a count outright. + @test BigInt(big(2)^53 + 3) != BigInt(Float64(big(2)^53 + 3)) + + @test_throws NP.UnsupportedRepresentationError NP.require_count(2.5) + @test_throws NP.UnsupportedRepresentationError NP.require_count(NaN) + @test_throws NP.UnsupportedRepresentationError NP.require_count(Inf) + @test_throws NP.UnsupportedRepresentationError NP.require_count(3 // 2) + @test_throws NP.UnsupportedRepresentationError NP.require_count("12") + @test NP.require_count(6 // 1) == 6 + end + + @testset "counts: overflow is named, not wrapped" begin + huge = typemax(Int64) + @test NP.checked_count_sum([1, 2, 3]) == 6 + @test NP.checked_count_sum(Int[]) == 0 + @test NP.checked_count_sum([Int32(2), Int64(3)]) == 5 + + err = try + NP.checked_count_sum([huge, 1]) + nothing + catch e + e + end + @test err isa NP.CountOverflowError + @test occursin(string(huge), sprint(showerror, err)) + + # Widening is the honest answer for a total a fixed-width integer cannot hold, + # and it says so in its type. + widened = NP.checked_count_sum([huge, 1]; on_overflow = :widen) + @test widened isa BigInt + @test widened == big(huge) + 1 + + @test_throws ArgumentError NP.checked_count_sum([1]; on_overflow = :shrug) + end + + @testset "proportions: a zero denominator has no proportion to report" begin + policy = NP.numeric_policy(; mode = :exact_counts) + @test NP.exact_relative_abundance(1, 4; policy) == 1 // 4 + @test NP.exact_relative_abundance(0, 4; policy) == 0 // 1 + @test NP.exact_relative_abundance(4, 4; policy) == 1 // 1 + + # The all-zero sample. Julia's own `//` would answer `1//0` and `0//0` is an + # exception only by accident: neither is a proportion, and this returns + # neither a number nor a throw -- it returns `nothing`, an unsuccessful state + # the presentation layer has to handle. + @test NP.exact_relative_abundance(0, 0; policy) === nothing + @test 1 // 0 isa Rational{Int} # the behaviour being refused + @test_throws NP.UnsupportedRepresentationError NP.exact_fraction(1, 0; policy) + + # Impossible inputs are programming errors, not data states, and throw. + @test_throws ArgumentError NP.exact_relative_abundance(5, 4; policy) + @test_throws ArgumentError NP.exact_relative_abundance(-1, 4; policy) + @test_throws ArgumentError NP.exact_relative_abundance(1, -4; policy) + end + + @testset "exact arithmetic keeps its exactness" begin + policy = NP.numeric_policy(; mode = :exact_counts) + thirds = [NP.exact_relative_abundance(1, 3; policy), + NP.exact_relative_abundance(1, 3; policy), + NP.exact_relative_abundance(1, 3; policy)] + @test NP.exact_rational_sum(thirds; policy) == 1 // 1 + + # Metamorphic: proportions of a table sum to exactly one, a property Float64 + # only approximates. This is the invariant the exact path exists to hold. + rng = Random.Xoshiro(20260922) + for _ in 1:50 + counts = rand(rng, 1:10_000, 25) + total = NP.checked_count_sum(counts) + parts = [NP.exact_relative_abundance(c, total; policy) for c in counts] + @test NP.exact_rational_sum(parts; policy) == 1 // 1 + end + + # An empty sum is not zero: nothing was added, so the sum has no value. + @test NP.exact_rational_sum(Rational{BigInt}[]; policy) === nothing + end + + @testset "exact arithmetic has a budget, and exhausting it is loud" begin + tiny = NP.numeric_policy(; mode = :exact_counts, max_denominator_bits = 32) + ok = [NP.exact_fraction(1, 2; policy = tiny), NP.exact_fraction(1, 3; policy = tiny)] + @test NP.exact_rational_sum(ok; policy = tiny) == 5 // 6 + + # Eleven primes: their lcm is 2.0e11, which is 38 bits of denominator, so the + # budget is exhausted partway through rather than at the first term. + primed = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31] + primes = [NP.exact_fraction(1, p; policy = tiny) for p in primed] + err = try + NP.exact_rational_sum(primes; policy = tiny) + nothing + catch e + e + end + @test err isa NP.ResourceLimitError + @test occursin("denominator", sprint(showerror, err)) + @test err.limit == 32 + @test err.observed > err.limit + + # The same sum is fine under the default budget, so the limit is a budget and + # not a bug. + @test NP.exact_rational_sum(primes; policy = NP.numeric_policy()) == + sum(1 // p for p in primed) + end + + @testset "precision: scoped, explicit, and reported back" begin + @test precision(BigFloat) == 256 # Julia's default, before anything here + + result, achieved = NP.with_precision(() -> BigFloat(1) / 3; bits = 512) + @test achieved == 512 + @test precision(result) == 512 + @test precision(BigFloat) == 256 # restored, not left behind + + @test_throws ArgumentError NP.with_precision(() -> 1; bits = 1) + + # Higher precision is not exactness: more bits still round. + exact_third = NP.exact_fraction(1, 3; policy = NP.numeric_policy()) + wide, _ = NP.with_precision(() -> BigFloat(1) / 3; bits = 512) + @test BigFloat(exact_third) != wide + end + + @testset "precision: two analyses do not share a setting" begin + # BigFloat precision is a process global, so isolation is by serialisation. + # Each task asserts what it observed *while it held the scope*; the lock is + # what makes those observations true rather than racy. + bits = [128, 256, 512, 1024, 2048, 4096] + observed = Vector{Int}(undef, length(bits)) + tasks = [Threads.@spawn begin + _, achieved = NP.with_precision(() -> precision(BigFloat); bits = b) + observed[i] = achieved + end for (i, b) in enumerate(bits)] + foreach(wait, tasks) + @test observed == bits + @test precision(BigFloat) == 256 + end + + @testset "values carry what they are, and the boundaries respect it" begin + count = NP.exact_value(42) + @test NP.is_exact(count) + @test NP.is_exact(NP.exact_value(3 // 4)) + @test !NP.is_exact(NP.approximate_value(0.5)) + @test !NP.is_exact(NP.rounded_value(0.5; digits = 2)) + + @test_throws NP.UnsupportedRepresentationError NP.exact_value(0.5) + @test_throws NP.UnsupportedRepresentationError NP.approximate_value(1) + @test_throws NP.UnsupportedRepresentationError NP.rounded_value(1; digits = 2) + + # Storage: small counts stay numbers, big ones and rationals become strings. + @test NP.to_storage(count) === 42 + @test NP.to_storage(NP.exact_value(big(2)^53 + 1)) == "9007199254740993" + @test NP.to_storage(NP.exact_value(3 // 4)) == "3/4" + @test NP.to_storage(NP.approximate_value(0.5)) === 0.5 + @test_throws NP.UnsupportedRepresentationError NP.to_storage(NP.approximate_value(NaN)) + + # Display says which kind it is showing, and rounds only here. + @test NP.to_display(count) == "42" + @test occursin("exact", NP.to_display(NP.exact_value(3 // 4))) + @test occursin("approximate", NP.to_display(NP.approximate_value(0.123456789))) + end + + @testset "the JSON boundary refuses loss rather than performing it" begin + @test NP.encode_json_number(7) === 7 + @test NP.encode_json_number(2^53 - 1) == 2^53 - 1 + @test NP.encode_json_number(big(2)^53) == "9007199254740992" + @test NP.encode_json_number(big(2)^53 + 1) == "9007199254740993" + @test NP.encode_json_number(3 // 4) == "3/4" + @test NP.encode_json_number(0.5) === 0.5 + @test NP.encode_json_number(BigFloat(1) / 3) isa String + + # The negative control the whole boundary exists for: the JavaScript client + # would receive 9007199254740992 for a count of 9007199254740993 if this were + # a JSON number. + lost = Float64(big(2)^53 + 1) + @test Int64(lost) != 9007199254740993 + @test NP.encode_json_number(big(2)^53 + 1) != lost + + @test_throws NP.UnsupportedRepresentationError NP.encode_json_number(NaN) + @test_throws NP.UnsupportedRepresentationError NP.encode_json_number(Inf) + @test_throws NP.UnsupportedRepresentationError NP.encode_json_number(-Inf) + @test_throws NP.UnsupportedRepresentationError NP.encode_json_number("12") + @test_throws NP.UnsupportedRepresentationError NP.encode_json_number(nothing) + + # And the encodings survive a round trip through JSON itself, since that is the + # trip they are for. + payload = JSON3.write(Dict("count" => NP.encode_json_number(big(2)^53 + 1), + "prop" => NP.encode_json_number(3 // 4), + "mean" => NP.encode_json_number(0.5))) + back = JSON3.read(payload) + @test back["count"] == "9007199254740993" + @test back["prop"] == "3/4" + @test back["mean"] == 0.5 + end +end