From c8c115b1fab37fa0fe486976f9c60c77039cbc97 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:11:49 +0000 Subject: [PATCH 1/9] fix(frontend): restore build and metadata tests Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- frontend/src/api/client.ts | 1 + frontend/src/api/types.ts | 26 +++++++++- .../src/components/PipelineStages.test.ts | 6 +-- .../src/components/differentialTable.test.ts | 4 +- frontend/src/components/differentialTable.ts | 49 ++++++++++++++++--- frontend/src/views/StudyView.tsx | 4 +- test/unit/test_numeric_policy.jl | 4 +- 7 files changed, 76 insertions(+), 18 deletions(-) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 98a97087..5b58aaa7 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -23,6 +23,7 @@ import type { ReferenceTreeDoc, ReferenceTreeSummary, ReferenceTreeDetail, AlignmentQC, TrimSettings, CladeCumulusSourcesResponse, CladeCumulusRequest, CladeCumulusResponse, CladeMoveValidationRequest, CladeMoveValidationResponse, + CladisticsSummary, CladisticsDetail, CladisticsCreateRequest, } from './types' import type { FigureDoc } from '../figure/types' diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index e668807d..593f7dc2 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -423,7 +423,10 @@ export interface DifferentialRow { prevalence: number } -export type DifferentialMethod = 'nb_glm' | 'clr_welch' | 'zinb' | 'hurdle_nb' +export type DifferentialMethod = + | 'nb_glm' | 'clr_welch' | 'zinb' | 'hurdle_nb' + | 'multinomial' | 'dirichlet_multinomial' + | 'ancom_bc' | 'aldex2' | 'songbird' /** Which row field holds the effect size, and what to call it. */ export interface DifferentialEffect { @@ -453,6 +456,16 @@ export interface DifferentialResult { replacement_delta?: number /** Null uses GBM's estimated concentration; positive values override it. */ bayesian_alpha?: number | null + /** multinomial and dirichlet_multinomial only. */ + mn_reference_taxon?: string | null + /** multinomial and dirichlet_multinomial only. */ + mn_penalty?: number + /** dirichlet_multinomial only. */ + dm_overdispersion_method?: 'mle' | 'mom' + /** Multinomial optimiser settings. */ + tolerance?: number + max_iter?: number + max_seconds?: number /** aldex2 only. */ mc_samples?: number /** aldex2 only. */ @@ -462,6 +475,17 @@ export interface DifferentialResult { n_taxa: number; n_tested: number; n_failed: number; n_boundary: number; n_filtered: number /** Safety note for two-part models. */ model_warning?: string + /** Diagnostics returned by the joint multinomial models. */ + model?: { + reference_taxon: string + penalty: number + n_parameters: number + iterations: number + converged: boolean + log_likelihood: number + runtime_seconds: number + n_taxa_fitted: number + } /** clr_welch only. */ zero_replacement?: { method: string diff --git a/frontend/src/components/PipelineStages.test.ts b/frontend/src/components/PipelineStages.test.ts index 0037ce55..2c46743a 100644 --- a/frontend/src/components/PipelineStages.test.ts +++ b/frontend/src/components/PipelineStages.test.ts @@ -24,11 +24,9 @@ test('advanced zero handling exposes both methods, bounded delta and nullable al expect(CONFIG_DESCRIPTIONS['analysis.differential.bayesian_alpha']).toContain('no posterior draws') }) -test('differential method enum lists all five compositional and count models', () => { +test('differential method enum lists exactly the methods accepted by the backend', () => { expect(CONFIG_TYPES['analysis.differential.method']).toEqual({ kind: 'enum', - options: ['nb_glm', 'clr_welch', 'ancom_bc', 'aldex2', 'songbird'], + options: ['nb_glm', 'clr_welch', 'zinb', 'hurdle_nb'], }) - expect(CONFIG_TYPES['analysis.differential.mc_samples']).toEqual({ kind: 'int' }) - expect(CONFIG_DESCRIPTIONS['analysis.differential.mc_samples']).toContain('Dirichlet') }) diff --git a/frontend/src/components/differentialTable.test.ts b/frontend/src/components/differentialTable.test.ts index 2bae9b2c..0090f3d1 100644 --- a/frontend/src/components/differentialTable.test.ts +++ b/frontend/src/components/differentialTable.test.ts @@ -99,7 +99,7 @@ describe('multinomial results', () => { test('the reference taxon is exported with an empty statistic set, never a substitute', () => { const lines = differentialCsv(mn).trimEnd().split('\r\n') - expect(lines[4].startsWith('Bacteroides,reference,0,0,,,,,,1,')).toBe(true) + expect(lines[4].startsWith('Bacteroides,reference,0,0,,,,,,,1,')).toBe(true) expect(effectValue(mn, mn.rows[1])).toBe(1.6) }) }) @@ -163,7 +163,7 @@ describe('advanced compositional methods (ancom_bc, aldex2, songbird)', () => { expect(effectValue(ax, ax.rows[0])).toBe(1.85) expect(configLine(ax)).toBe('method aldex2, mc_samples 128, seed 123, min_prevalence 0.05; estimate is an ALDEx2 standardized CLR effect size, not a fold change') const csvRows = differentialCsv(ax).trimEnd().split('\r\n') - expect(csvRows[4]).toBe('Bacteroides,ok,,1.85,,5.545,2.9e-8,5.8e-8,,1,') + expect(csvRows[4]).toBe('Bacteroides,ok,,1.85,,5.545,2.9e-8,5.8e-8,,,1,') }) test('songbird reports CLR-centred multinomial differential metadata', () => { diff --git a/frontend/src/components/differentialTable.ts b/frontend/src/components/differentialTable.ts index 2743553e..fd4b3980 100644 --- a/frontend/src/components/differentialTable.ts +++ b/frontend/src/components/differentialTable.ts @@ -46,13 +46,48 @@ export function formatStat(value: number | null): string { */ export function configLine(result: DifferentialResult): string { const c = result.config - if (c.method !== 'clr_welch') - return `method ${c.method}, offset ${c.offset}, min_prevalence ${c.min_prevalence}` - const replacement = c.replacement_method ?? 'bayesian_multiplicative' - const alpha = c.bayesian_alpha === null || c.bayesian_alpha === undefined - ? 'estimated' : String(c.bayesian_alpha) - return `method clr_welch, replacement_method ${replacement}, replacement_delta ${c.replacement_delta}, ` + - `bayesian_alpha ${alpha}, min_prevalence ${c.min_prevalence}; estimate is a CLR difference, not a fold change` + + if (c.method === 'multinomial' || c.method === 'dirichlet_multinomial') { + const settings = [`method ${c.method}`] + if (c.method === 'dirichlet_multinomial' && c.dm_overdispersion_method) + settings.push(`overdispersion ${c.dm_overdispersion_method}`) + if (c.mn_reference_taxon) settings.push(`reference ${c.mn_reference_taxon}`) + if (c.mn_penalty !== undefined) settings.push(`penalty ${c.mn_penalty}`) + settings.push(`min_prevalence ${c.min_prevalence}`) + return `${settings.join(', ')}; estimates are log2 fold changes against the reference taxon` + } + + if (c.method === 'clr_welch') { + const replacement = c.replacement_method ?? 'bayesian_multiplicative' + const settings = [ + `method clr_welch`, + `replacement_method ${replacement}`, + `replacement_delta ${c.replacement_delta}`, + ] + // CZM has no Bayesian concentration parameter; do not claim one was used. + if (replacement === 'bayesian_multiplicative') { + const alpha = c.bayesian_alpha === null || c.bayesian_alpha === undefined + ? 'estimated' : String(c.bayesian_alpha) + settings.push(`bayesian_alpha ${alpha}`) + } + settings.push(`min_prevalence ${c.min_prevalence}`) + return `${settings.join(', ')}; estimate is a CLR difference, not a fold change` + } + + if (c.method === 'ancom_bc') + return `method ancom_bc, min_prevalence ${c.min_prevalence}; effect is bias-corrected log2 fold change (Lin & Peddada 2020)` + + if (c.method === 'aldex2') + return `method aldex2, mc_samples ${c.mc_samples}, seed ${c.seed}, min_prevalence ${c.min_prevalence}; estimate is an ALDEx2 standardized CLR effect size, not a fold change` + + if (c.method === 'songbird') + return `method songbird, min_prevalence ${c.min_prevalence}; estimate is a CLR-centred multinomial differential, not an absolute fold change` + + const settings = [`method ${c.method}`] + if (c.offset !== undefined) settings.push(`offset ${c.offset}`) + if (c.method === 'nb_glm' && c.dispersion !== undefined) settings.push(`dispersion ${c.dispersion}`) + settings.push(`min_prevalence ${c.min_prevalence}`) + return settings.join(', ') } /** The effect size of one row, from the field the result names in `effect`. */ diff --git a/frontend/src/views/StudyView.tsx b/frontend/src/views/StudyView.tsx index 89f32231..553d948d 100644 --- a/frontend/src/views/StudyView.tsx +++ b/frontend/src/views/StudyView.tsx @@ -15,7 +15,7 @@ import { ConfigAccordion } from '../components/ConfigAccordion' import { TreesPanel } from '../components/TreesPanel' import { PlacementPanel } from '../components/PlacementPanel' import { EvidenceModeToggle } from '../components/EvidenceModeToggle' -import { CladeCumulusPanel } from '../components/CladeCumulusPanel' +import { CladeCumulusPanel as EvidenceCladeCumulusPanel } from '../components/CladeCumulusPanel' import { ReportPanel } from '../components/ReportPanel' import { useNavRefresh } from '../hooks/useNavRefresh' import { CladeCumulusPanel } from '../cladistics/CladeCumulusPanel' @@ -304,7 +304,7 @@ export function StudyView() { {view === 'trees' && <> - {evidenceMode && } + {evidenceMode && } setTreesVersion(v => v + 1)} /> } diff --git a/test/unit/test_numeric_policy.jl b/test/unit/test_numeric_policy.jl index 2f64ff8c..5c2cdf5b 100644 --- a/test/unit/test_numeric_policy.jl +++ b/test/unit/test_numeric_policy.jl @@ -51,8 +51,8 @@ end tight = numeric_policy(mode=:exact_counts, max_denominator_bits=32) @test_throws ResourceLimitError exact_fraction(1, big(2)^40; policy=tight) @test_throws ResourceLimitError exact_fraction(big(2)^40, 1; policy=tight) - # Each addend fits the 32-bit budget on its own; their sum 1/2^31 + 1/3 needs - # 33 bits, so the limit must fire on the running total, not only on inputs. + # Each addend fits the 32-bit budget on its own; their reduced sum has a + # 62-bit denominator, so the limit must fire on the running total. @test_throws ResourceLimitError exact_rational_sum( [big(1)//big(2)^31, big(1)//(big(2)^31 - 1)]; policy=tight) end From fd342911d5b89af6dc114ef943d6e8537782b035 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:17:39 +0000 Subject: [PATCH 2/9] test(stats): verify 1024-bit references independently Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- test/references/check_high_precision_references.py | 2 +- test/unit/test_high_precision.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/references/check_high_precision_references.py b/test/references/check_high_precision_references.py index 3d644c8f..c63ffa23 100644 --- a/test/references/check_high_precision_references.py +++ b/test/references/check_high_precision_references.py @@ -14,7 +14,7 @@ import re import sys -PRECISION = 220 +PRECISION = 460 def references() -> dict[str, str]: diff --git a/test/unit/test_high_precision.jl b/test/unit/test_high_precision.jl index 1b903e08..9f4c9e96 100644 --- a/test/unit/test_high_precision.jl +++ b/test/unit/test_high_precision.jl @@ -10,9 +10,9 @@ const HP_LABELS = ["A", "B"] const HP_FEATURES = ["f1", "f2", "f3"] const REF_LN2 = "0.6931471805599453094172321214581765680755001343602552541206800094933936219696947156058633269964186875420014810205706857336855202357581305570326707516350759619307275708283714351903070386238916734711233501153644979552391204751726815749320651555247341395258829504530070953263666426541042391578149520437404303855008019441706416715186447128399681717845469570262716310645461502572074024816377733896385506952606683411372738737229289564935470257626520988596932019650586" -const REF_LN3 = "1.098612288668109691395245236922525704647490557822749451734694333637494293218608966873615754813732088787970029065957865742368004225930519821052801870767277410603162769183381367179373698844360959903742570316795911521145592" -const REF_TWO_THIRDS = "0.6666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666667" -const REF_H_1234 = "1.279854225833667467182185762149429928131354186921692886392239591382930699742075074568049151649059984163575294301928203177013603961545520253317776056233522504371230806742168835499680746549033237086147384098562106321046048" +const REF_LN3 = "1.098612288668109691395245236922525704647490557822749451734694333637494293218608966873615754813732088787970029065957865742368004225930519821052801870767277410603162769183381367179373698844360959903742570316795911521145591917750671347054940166775580222203170252946897560690106521505642868138036317373298577782366991654792131818149020030103823630122248652748198225991097452490896458053467008845965085748444119018857087647494867079613085829411602166121184001409826" +const REF_TWO_THIRDS = "0.6666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666667" +const REF_H_1234 = "1.279854225833667467182185762149429928131354186921692886392239591382930699742075074568049151649059984163575294301928203177013603961545520253317776056233522504371230806742168835499680746549033237086147384098562106321046048112868339308649672681189135461287570216937288791515535715833594380020178552966375097993963900928200120047744717834903206362014300458629735272704036751340598445797090470318251133760850603018005627582297102594277585911107434593371009711254916" function test_bigfloat_close(actual, reference, bits, ulps=16) setprecision(BigFloat, bits) do From 185e51e3355520f6ded4404289ff31c4b19badab Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:09:05 +0000 Subject: [PATCH 3/9] ci: expose validated statistics failure context Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/ci.yml | 21 ++++++++++++++++++++- test/unit/test_validated_statistics.jl | 3 +++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bad3b7b..ba7d476c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -404,7 +404,26 @@ jobs: status="${PIPESTATUS[0]}" if [ "$status" -ne 0 ]; then enc() { head -c 3500 | sed -e 's/%/%25/g' -e 's/\r/%0D/g' | awk '{printf "%s%%0A", $0}'; } - echo "::error title=Validated-statistics tests failed::$(tail -n 120 /tmp/validated-statistics-tests.log | enc)" + awk ' + /Test Summary:/ { left = 0 } + /(Error: validated run statistics|Test Failed at|Error During Test at)/ || (left == 0 && /Got exception outside of a @test/) { n++; left = 60 } + left > 0 { print > ("/tmp/validated-failure-" n ".txt"); left-- } + ' /tmp/validated-statistics-tests.log + shown=0 + for f in $(ls /tmp/validated-failure-*.txt 2>/dev/null | sort -t- -k3 -n); do + shown=$((shown + 1)) + [ "$shown" -gt 8 ] && break + echo "::error title=Validated-statistics failure ${shown}::$(enc < "$f")" + done + awk ' + /Test Summary:/ { print; p = index($0, "Pass"); pe = (p ? p + 3 : 0); t = index($0, "Total"); next } + pe && t && index($0, "|") && substr($0, pe + 1, t - pe - 1) ~ /[0-9]/ { print; next } + /ERROR:|Some tests did not pass/ { print } + ' /tmp/validated-statistics-tests.log | tail -n 60 > /tmp/validated-statistics-summary.txt + echo "::error title=Validated-statistics tests failed (summary)::$(enc < /tmp/validated-statistics-summary.txt)" + if [ "$shown" -eq 0 ]; then + echo "::error title=Validated-statistics tests failed (tail)::$(tail -c 3500 /tmp/validated-statistics-tests.log | enc)" + fi fi exit "$status" diff --git a/test/unit/test_validated_statistics.jl b/test/unit/test_validated_statistics.jl index c19bee2c..c7c81f79 100644 --- a/test/unit/test_validated_statistics.jl +++ b/test/unit/test_validated_statistics.jl @@ -177,6 +177,9 @@ end request = Dict("numeric_policy" => Dict("mode" => "exact_counts"), "table" => "merged", "taxonomic_level" => "Genus") response = _vs_post(path, request) + if response.status != 200 + @error "validated run statistics request failed" status=response.status body=String(response.body) + end @test response.status == 200 result = JSON3.read(String(response.body)) @test result.status == "partial" # Empty has undefined proportions. From 2495b9eee50d6961ec9d7f34d63a471d39b98770 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:21:39 +0000 Subject: [PATCH 4/9] fix: aggregate validated run counts with BigInt Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- src/server/routes/validated_statistics.jl | 39 +++++++++++++++-------- test/unit/test_exact_summaries.jl | 4 +-- test/unit/test_validated_statistics.jl | 9 +++++- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba7d476c..1bd8207c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -406,7 +406,7 @@ jobs: enc() { head -c 3500 | sed -e 's/%/%25/g' -e 's/\r/%0D/g' | awk '{printf "%s%%0A", $0}'; } awk ' /Test Summary:/ { left = 0 } - /(Error: validated run statistics|Test Failed at|Error During Test at)/ || (left == 0 && /Got exception outside of a @test/) { n++; left = 60 } + /(Error: (validated run statistics|numeric input did not fail closed|exponent numeric token did not fail closed)|Test Failed at|Error During Test at)/ || (left == 0 && /Got exception outside of a @test/) { n++; left = 60 } left > 0 { print > ("/tmp/validated-failure-" n ".txt"); left-- } ' /tmp/validated-statistics-tests.log shown=0 diff --git a/src/server/routes/validated_statistics.jl b/src/server/routes/validated_statistics.jl index ce214c7d..328188d0 100644 --- a/src/server/routes/validated_statistics.jl +++ b/src/server/routes/validated_statistics.jl @@ -180,8 +180,9 @@ function _validated_integer_sample_columns(con, table::String, columns::Vector{S samples end -"""Read and aggregate an integer-safe run table. Only GROUP BY is done in SQL; -all count addition is Julia BigInt. No SQL SUM and no Float64 matrix is used.""" +"""Read and aggregate an integer-safe run table. SQL only selects bounded rows; +feature grouping and all count addition happen in Julia BigInt. No SQL SUM or +Float64 matrix is used.""" function _reject_float_count_filters(sample_columns::Vector{String}, params::Dict{String,String}) for sample in sample_columns, prefix in ("col_min.", "col_max.") key = prefix * sample @@ -218,31 +219,43 @@ function _validated_duckdb_counts(con, table::String, columns::Vector{String}; "COALESCE(NULLIF(CAST($(_sql_identifier(taxon)) AS VARCHAR), ''), 'Unassigned')" end count_columns = join([_sql_identifier(c) for c in samples], ", ") + # Bound the raw rows before materialising the result. SQL deliberately does + # no SUM: feature-level addition is performed below with BigInt so HUGEINT + # values cannot overflow while duplicate taxonomy rows are aggregated. + max_rows = div(MAX_CELLS, length(samples)) sql = "SELECT $feature_expression AS feature, $count_columns FROM $(_sql_identifier(table)) " * - "$where_clause GROUP BY $feature_expression ORDER BY feature LIMIT $(MAX_CELLS + 1)" + "$where_clause ORDER BY feature LIMIT $(max_rows + 1)" frame = DataFrame(DBInterface.execute(con, sql, where_params)) - nrow(frame) <= MAX_CELLS || throw(ResourceLimitError("features", MAX_CELLS, nrow(frame))) + observed_cells = nrow(frame) * length(samples) + observed_cells <= MAX_CELLS || throw(ResourceLimitError( + "input table cells", MAX_CELLS, observed_cells)) nrow(frame) > 0 || throw(ArgumentError("no rows remain after filtering")) - nrow(frame) * length(samples) <= MAX_CELLS || throw(ResourceLimitError( - "table cells", MAX_CELLS, nrow(frame) * length(samples))) - features = Vector{String}(undef, nrow(frame)) - counts = Matrix{BigInt}(undef, nrow(frame), length(samples)) + aggregated = Dict{String,Vector{BigInt}}() for (i, row) in enumerate(eachrow(frame)) ismissing(row.feature) && throw(UnsupportedRepresentationError( "a non-missing feature label", "missing", "validated run statistics row $i")) - features[i] = string(row.feature) + feature = string(row.feature) + feature_counts = get!(aggregated, feature) do + zeros(BigInt, length(samples)) + end for (j, sample) in enumerate(samples) value = row[Symbol(sample)] ismissing(value) && throw(UnsupportedRepresentationError( - "a non-missing integer count", "NULL", "feature '$(features[i])', sample '$sample'")) + "a non-missing integer count", "NULL", "feature '$feature', sample '$sample'")) (value isa Integer && !(value isa Bool)) || throw(UnsupportedRepresentationError( - "an integer count", repr(value), "feature '$(features[i])', sample '$sample'")) + "an integer count", repr(value), "feature '$feature', sample '$sample'")) value >= 0 || throw(ArgumentError( - "negative count $value at feature '$(features[i])', sample '$sample'")) - counts[i, j] = BigInt(value) # every addition later is exact BigInt + "negative count $value at feature '$feature', sample '$sample'")) + feature_counts[j] += BigInt(value) end end + + features = sort!(collect(keys(aggregated))) + counts = Matrix{BigInt}(undef, length(features), length(samples)) + for (i, feature) in enumerate(features) + counts[i, :] = aggregated[feature] + end ExactSummaries.validate_coordinates(counts, samples, features) counts, samples, features end diff --git a/test/unit/test_exact_summaries.jl b/test/unit/test_exact_summaries.jl index e1ace22d..71d8c004 100644 --- a/test/unit/test_exact_summaries.jl +++ b/test/unit/test_exact_summaries.jl @@ -14,7 +14,7 @@ const EXACT_FEATURES = ["f1", "f2", "f3"] @test !s.approximate @test s.samples[1].total == 3 @test s.samples[1].proportions == fill(big(1)//3, 3) - @test s.samples[2].proportions == Union{Rational{BigInt},Nothing}[big(1), nothing, nothing] + @test s.samples[2].proportions == Union{Rational{BigInt},Nothing}[big(1), big(0), big(0)] @test length(s.groups) == 2 @test s.groups[1].label == "g1" @test s.groups[2].counts == BigInt[2, 0, 0] @@ -28,7 +28,7 @@ const EXACT_FEATURES = ["f1", "f2", "f3"] @test storage["input_status"] == "exact" @test storage["samples"][1]["proportions"] == ["1/3", "1/3", "1/3"] @test storage["samples"][1]["total"] == 3 - @test storage["samples"][2]["proportions"][2] === nothing + @test storage["samples"][2]["proportions"][2] == "0/1" @test length(jcs_sha256(storage)) == 64 display = summary_to_display(s; digits=8) @test occursin("exact", display) diff --git a/test/unit/test_validated_statistics.jl b/test/unit/test_validated_statistics.jl index c7c81f79..8b95601d 100644 --- a/test/unit/test_validated_statistics.jl +++ b/test/unit/test_validated_statistics.jl @@ -96,12 +96,19 @@ end payload = merge(_VS_EXACT_BODY, Dict("persist" => false, "counts" => [[bad, 1], [1, 0], [1, 0]])) response = _vs_post("/api/v1/statistics/summaries", payload) + if response.status != 422 + @error "numeric input did not fail closed" value=repr(bad) status=response.status body=String(response.body) + end @test response.status == 422 @test JSON3.read(String(response.body)).error == "invalid_numeric_input" end # Exponent notation is a float token even though its value is integral. raw = """{"numeric_policy":{"mode":"exact_counts"},"sample_labels":["A"],"feature_labels":["f"],"counts":[[1e0]]}""" - @test _vs_post("/api/v1/statistics/summaries", raw; raw=true).status == 422 + exponent_response = _vs_post("/api/v1/statistics/summaries", raw; raw=true) + if exponent_response.status != 422 + @error "exponent numeric token did not fail closed" status=exponent_response.status body=String(exponent_response.body) + end + @test exponent_response.status == 422 negative_string = merge(_VS_EXACT_BODY, Dict("persist" => false, "counts" => [["-1", 1], [1, 0], [1, 0]])) @test _vs_post("/api/v1/statistics/summaries", negative_string).status == 422 From 05aabac848fb6d667b2c2c764112f2fb4fde3b94 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:36:03 +0000 Subject: [PATCH 5/9] fix: preserve exact JSON count token validation Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- src/server/routes/validated_statistics.jl | 88 ++++++++++++++++++++++- test/unit/test_validated_statistics.jl | 17 ++--- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/server/routes/validated_statistics.jl b/src/server/routes/validated_statistics.jl index 328188d0..1bb4fd96 100644 --- a/src/server/routes/validated_statistics.jl +++ b/src/server/routes/validated_statistics.jl @@ -38,11 +38,97 @@ function _statistics_guard(f::Function) end end +function _json_quoted_end(bytes, start::Int) + i = start + 1 + while i <= length(bytes) + if bytes[i] == 0x5c # escaped byte + i += 2 + elseif bytes[i] == 0x22 + return i + else + i += 1 + end + end + nothing +end + +function _json_skip_whitespace(bytes, i::Int) + while i <= length(bytes) && bytes[i] in (0x20, 0x09, 0x0a, 0x0d) + i += 1 + end + i +end + +function _reject_noninteger_count_tokens(bytes, start::Int) + depth = 0 + i = start + while i <= length(bytes) + byte = bytes[i] + if byte == 0x22 + end_quote = _json_quoted_end(bytes, i) + isnothing(end_quote) && return nothing # JSON3 reports the syntax error. + i = end_quote + 1 + elseif byte == 0x5b + depth += 1 + i += 1 + elseif byte == 0x5d + depth -= 1 + i += 1 + depth == 0 && return nothing + elseif byte == 0x2d || 0x30 <= byte <= 0x39 + token_start = i + while i <= length(bytes) && + (bytes[i] in (0x2b, 0x2d, 0x2e, 0x45, 0x65) || 0x30 <= bytes[i] <= 0x39) + i += 1 + end + token = String(collect(bytes[token_start:i-1])) + occursin(r"[.eE]", token) && throw(UnsupportedRepresentationError( + "an integer JSON count token", token, + "validated statistics; decimal and exponent tokens are not exact integers")) + else + i += 1 + end + end + nothing +end + +# JSON3 may normalise integral decimal/exponent tokens to an integer value, +# losing the source representation. Inspect the raw `counts` array first so +# `1.0` and `1e0` cannot silently pass an exact-count boundary as `1`. +function _reject_approximate_count_tokens(raw::String) + bytes = codeunits(raw) + i = 1 + while i <= length(bytes) + if bytes[i] != 0x22 + i += 1 + continue + end + end_quote = _json_quoted_end(bytes, i) + isnothing(end_quote) && return nothing # JSON3 reports the syntax error. + key = try + JSON3.read(String(collect(bytes[i:end_quote]))) + catch + nothing + end + after_key = _json_skip_whitespace(bytes, end_quote + 1) + if key == "counts" && after_key <= length(bytes) && bytes[after_key] == 0x3a + value_start = _json_skip_whitespace(bytes, after_key + 1) + if value_start <= length(bytes) && bytes[value_start] == 0x5b + _reject_noninteger_count_tokens(bytes, value_start) + end + end + i = end_quote + 1 + end + nothing +end + function _statistics_body(req) length(req.body) <= _STATISTICS_REQUEST_LIMIT || throw(ResourceLimitError( "request bytes", _STATISTICS_REQUEST_LIMIT, length(req.body))) + raw = String(req.body) + _reject_approximate_count_tokens(raw) parsed = try - JSON3.read(String(req.body)) + JSON3.read(raw) catch throw(ArgumentError("request body is not valid JSON")) end diff --git a/test/unit/test_validated_statistics.jl b/test/unit/test_validated_statistics.jl index 8b95601d..e4c64d70 100644 --- a/test/unit/test_validated_statistics.jl +++ b/test/unit/test_validated_statistics.jl @@ -92,23 +92,16 @@ end @testset "Numeric JSON representations fail closed" begin _vs_with_root() do _ - for bad in (1.0, 1.5, -1, 9007199254740993) - payload = merge(_VS_EXACT_BODY, Dict("persist" => false, - "counts" => [[bad, 1], [1, 0], [1, 0]])) - response = _vs_post("/api/v1/statistics/summaries", payload) + # Raw bodies preserve numeric syntax; JSON3.write may normalize 1.0 to 1. + for bad in ("1.0", "1.5", "-1", "9007199254740993", "1e0") + raw = """{"numeric_policy":{"mode":"exact_counts"},"sample_labels":["A","B"],"feature_labels":["f"],"counts":[[$bad,1]],"persist":false}""" + response = _vs_post("/api/v1/statistics/summaries", raw; raw=true) if response.status != 422 - @error "numeric input did not fail closed" value=repr(bad) status=response.status body=String(response.body) + @error "numeric input did not fail closed" token=bad status=response.status body=String(response.body) end @test response.status == 422 @test JSON3.read(String(response.body)).error == "invalid_numeric_input" end - # Exponent notation is a float token even though its value is integral. - raw = """{"numeric_policy":{"mode":"exact_counts"},"sample_labels":["A"],"feature_labels":["f"],"counts":[[1e0]]}""" - exponent_response = _vs_post("/api/v1/statistics/summaries", raw; raw=true) - if exponent_response.status != 422 - @error "exponent numeric token did not fail closed" status=exponent_response.status body=String(exponent_response.body) - end - @test exponent_response.status == 422 negative_string = merge(_VS_EXACT_BODY, Dict("persist" => false, "counts" => [["-1", 1], [1, 0], [1, 0]])) @test _vs_post("/api/v1/statistics/summaries", negative_string).status == 422 From b6b5c3799a4f23e53cecff410adaa9b59f1c93ca Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:57:25 +0000 Subject: [PATCH 6/9] fix: resolve clade cumulus integration test failures Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- src/analysis/clade_cumulus.jl | 15 ++++++++++----- src/server/routes/clade_cumulus.jl | 8 ++++---- test/unit/test_clade_cumulus.jl | 19 ++++++++++++------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/analysis/clade_cumulus.jl b/src/analysis/clade_cumulus.jl index a702fde2..3a0136d2 100644 --- a/src/analysis/clade_cumulus.jl +++ b/src/analysis/clade_cumulus.jl @@ -203,7 +203,9 @@ function parse_newick(src::AbstractString; warnings::Vector{String} = String[]): end function read_label_and_quoted()::Tuple{String,Bool} - skip_ws_and_leading_comments!() + while pos[] <= n && bytes[pos[]] in (UInt8(' '), UInt8('\t'), UInt8('\r'), UInt8('\n')) + pos[] += 1 + end pos[] > n && return ("", false) if bytes[pos[]] == UInt8('\'') pos[] += 1 @@ -260,6 +262,9 @@ function parse_newick(src::AbstractString; warnings::Vector{String} = String[]): nhx_sup = Ref{Union{Float64,Nothing}}(nothing) edge_num = Ref{Union{Int,Nothing}}(nothing) + # Comments directly after ')' may carry NHX support for the internal node; + # consume them with capture enabled before reading its optional label. + skip_ws_and_leading_comments!(capture_nhx = nhx_sup, capture_edge = edge_num) raw_lbl, was_quoted = read_label_and_quoted() skip_ws_and_leading_comments!(capture_nhx = nhx_sup, capture_edge = edge_num) @@ -991,7 +996,7 @@ function _graft_jplace_queries( for q in jd.queries q.status === :valid && !isempty(q.candidates) || continue - total_w = sum((c.weight for c in q.candidates); init = 0.0) + total_w = sum((c.like_weight for c in q.candidates); init = 0.0) total_w > 0.0 || continue multi = length(q.candidates) > 1 for (qname, mult) in q.names @@ -1000,7 +1005,7 @@ function _graft_jplace_queries( tx = get(asv_deepest_taxon, qname, nothing) tconf = get(asv_tax_conf, qname, 1.0) for (ci, cand) in enumerate(q.candidates) - share = cand.weight / total_w + share = cand.like_weight / total_w tip_id = multi ? string(qname, "@edge", cand.edge_num, "#", ci) : qname if multi asv_reads[tip_id] = get(asv_reads, tip_id, 0.0) + base_cnt * share @@ -1009,9 +1014,9 @@ function _graft_jplace_queries( elseif !haskey(asv_reads, tip_id) asv_reads[tip_id] = base_cnt end - query_lwr[tip_id] = cand.weight + query_lwr[tip_id] = cand.like_weight push!(get!(by_edge, cand.edge_num, _GraftedQuerySpec[]), - _GraftedQuerySpec(tip_id, qname, share, max(0.0, cand.distal_length), max(0.0, cand.pendant_length), cand.weight)) + _GraftedQuerySpec(tip_id, qname, share, max(0.0, cand.distal_len), max(0.0, cand.pendant_len), cand.like_weight)) end end end diff --git a/src/server/routes/clade_cumulus.jl b/src/server/routes/clade_cumulus.jl index 45825b61..d65ef535 100644 --- a/src/server/routes/clade_cumulus.jl +++ b/src/server/routes/clade_cumulus.jl @@ -200,7 +200,7 @@ end return json_error(400, "invalid_json", "Request body must be valid JSON") end - table = strip(string(get(body, :table, "merged"))) + table = String(strip(string(get(body, :table, "merged")))) _valid_name(table) || return json_error(400, "invalid_table", "Invalid table name '$table'") mode_str = lowercase(strip(string(get(body, :mode, "combined")))) @@ -268,13 +268,13 @@ end jd_div = MetaManifold.CladeCumulus.parse_jplace(jplace_str) for q in jd_div.queries q.status === :valid && !isempty(q.candidates) || continue - tw = sum((c.weight for c in q.candidates); init = 0.0) + tw = sum((c.like_weight for c in q.candidates); init = 0.0) tw > 0.0 || continue multi = length(q.candidates) > 1 for (qname, _) in q.names for (ci, cand) in enumerate(q.candidates) tid = multi ? string(qname, "@edge", cand.edge_num, "#", ci) : qname - tip_weights[tid] = cand.weight / tw + tip_weights[tid] = cand.like_weight / tw end end end @@ -391,7 +391,7 @@ end (isempty(source_id) || isempty(target_id)) && return json_error(400, "missing_clade_ids", "Both sourceId and targetId are required") - table = strip(string(get(body, :table, "merged"))) + table = String(strip(string(get(body, :table, "merged")))) _valid_name(table) || return json_error(400, "invalid_table", "Invalid table name '$table'") mode_str = lowercase(strip(string(get(body, :mode, "combined")))) diff --git a/test/unit/test_clade_cumulus.jl b/test/unit/test_clade_cumulus.jl index da09bc18..28499525 100644 --- a/test/unit/test_clade_cumulus.jl +++ b/test/unit/test_clade_cumulus.jl @@ -134,7 +134,7 @@ _cc_req(method::String, path::String, body::String = "") = "placements" => [ OrderedDict("n" => ["Q_split"], "p" => [[0, -100.0, 0.60, 0.01, 0.02], [1, -100.4, 0.40, 0.01, 0.03]]), OrderedDict("nm" => [["Q_borderline", 1]], "p" => [[3, -98.0, 0.85, 0.05, 0.02], [2, -101.0, 0.15, 0.05, 0.02]]), - OrderedDict("n" => ["Q_strong"], "p" => [[3, -97.0, 0.98, 0.02, 0.01]]), + OrderedDict("n" => ["Q_strong"], "p" => [[3, -97.0, 0.98, 0.02, 0.01], [2, -101.0, 0.02, 0.02, 0.01]]), OrderedDict("n" => ["Q_bad_sum"], "p" => [[0, -100.0, 0.40, 0.01, 0.01]]), ], )) @@ -252,8 +252,8 @@ done echo "(MMTIP_000001:0.05,(MMTIP_000002:0.03,MMTIP_000003:0.04)0.910:0.12,MMTIP_000004:0.08);" > "$OUT" """) chmod(mock_ft, 0o755) - old_ft = get(PH.Tools._tools, "fasttree", nothing) - PH.Tools._tools["fasttree"] = mock_ft + old_ft = get(MetaManifold.Tools._tools, "fasttree", nothing) + MetaManifold.Tools._tools["fasttree"] = mock_ft try s4 = [ "ASV_1" => "ACGTACGTACGTACGTACGT", @@ -267,9 +267,9 @@ echo "(MMTIP_000001:0.05,(MMTIP_000002:0.03,MMTIP_000003:0.04)0.910:0.12,MMTIP_0 r["sha256"] == bytes2hex(open(sha256, mock_ft)), ft_res.tool_records) finally if old_ft === nothing - delete!(PH.Tools._tools, "fasttree") + delete!(MetaManifold.Tools._tools, "fasttree") else - PH.Tools._tools["fasttree"] = old_ft + MetaManifold.Tools._tools["fasttree"] = old_ft end end end @@ -308,8 +308,13 @@ echo "(MMTIP_000001:0.05,(MMTIP_000002:0.03,MMTIP_000003:0.04)0.910:0.12,MMTIP_0 db_path = joinpath(merge_dir, "results.duckdb") db = DuckDB.DB(db_path) con = DBInterface.connect(db) - DuckDB.register_data_frame(con, df, "src_df") - DBInterface.execute(con, "CREATE TABLE merged AS SELECT * FROM src_df") + DBInterface.execute(con, """CREATE TABLE merged ( + SeqName VARCHAR, Domain VARCHAR, Genus VARCHAR, Sequence VARCHAR, + SampleA BIGINT, SampleB BIGINT)""") + for row in eachrow(df) + DBInterface.execute(con, "INSERT INTO merged VALUES (?, ?, ?, ?, ?, ?)", + [row.SeqName, row.Domain, row.Genus, row.Sequence, row.SampleA, row.SampleB]) + end DBInterface.close!(con) close(db) before_db_sha = bytes2hex(open(sha256, db_path)) From cadd957369d5d83de7ee2f823f75a5049a0d8629 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:10:15 +0000 Subject: [PATCH 7/9] fix: align tree route and graft field names Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- src/analysis/clade_cumulus.jl | 6 +++--- src/server/routes/clade_cumulus.jl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analysis/clade_cumulus.jl b/src/analysis/clade_cumulus.jl index 3a0136d2..3a996274 100644 --- a/src/analysis/clade_cumulus.jl +++ b/src/analysis/clade_cumulus.jl @@ -1026,8 +1026,8 @@ function _graft_jplace_queries( function walk(u::PhyloNode)::PhyloNode rebuilt_kids = PhyloNode[walk(ch) for ch in u.children] - cur = PhyloNode(u.id, u.name, u.branch_length, u.support, u.edge_num, rebuilt_kids) - specs = u.edge_num === nothing ? nothing : get(by_edge, u.edge_num, nothing) + cur = PhyloNode(u.id, u.name, u.branch_length, u.support, u.edge_number, rebuilt_kids) + specs = u.edge_number === nothing ? nothing : get(by_edge, u.edge_number, nothing) (specs === nothing || isempty(specs)) && return cur # Order grafts from most distal (closest to `cur`, smallest `distal`) to most proximal. @@ -1045,7 +1045,7 @@ function _graft_jplace_queries( sub = PhyloNode(string("jplace_clade#", seq_auto[]), "", 0.0, 1.0, nothing, PhyloNode[sub, qtip]) end rem_len = max(0.0, total_len - prev_distal) - PhyloNode(sub.id, sub.name, rem_len, cur.support, cur.edge_num, sub.children) + PhyloNode(sub.id, sub.name, rem_len, cur.support, cur.edge_number, sub.children) end walk(base_root) diff --git a/src/server/routes/clade_cumulus.jl b/src/server/routes/clade_cumulus.jl index d65ef535..be1aaa84 100644 --- a/src/server/routes/clade_cumulus.jl +++ b/src/server/routes/clade_cumulus.jl @@ -93,7 +93,7 @@ function _resolve_tree_inputs_from_request( source_digests::OrderedDict{String,String}, ) tree_source = lowercase(strip(string(get(body, :treeSource, "auto")))) - tree_file = strip(string(get(body, :treeFile, ""))) + tree_file = String(strip(string(get(body, :treeFile, "")))) inline_newick = strip(string(get(body, :newick, ""))) if tree_source == "study_tree" || (!isempty(tree_file) && tree_source == "auto") From d17e6c50f257edd23c62aeb0a72d28b85ff7329e Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:27:07 +0000 Subject: [PATCH 8/9] fix: clear Julia suite regressions Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- src/analysis/clade_cumulus.jl | 4 ++++ src/server/routes/clade_cumulus.jl | 25 +++++++++---------------- test/unit/test_validated_statistics.jl | 12 ++++++------ 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/analysis/clade_cumulus.jl b/src/analysis/clade_cumulus.jl index 3a996274..6c6d6f69 100644 --- a/src/analysis/clade_cumulus.jl +++ b/src/analysis/clade_cumulus.jl @@ -1821,6 +1821,10 @@ function _reaggregate_after_move( end end + # Match `clade_faith_pd`: an active non-root clade contributes its own + # incoming branch in addition to the branches accumulated below it. + !is_root && any_active && (sub_pd += max(0.0, u.branch_length)) + dom_tx = nothing dom_n = 0 for (tx, c) in merged_taxa diff --git a/src/server/routes/clade_cumulus.jl b/src/server/routes/clade_cumulus.jl index be1aaa84..481ba016 100644 --- a/src/server/routes/clade_cumulus.jl +++ b/src/server/routes/clade_cumulus.jl @@ -71,19 +71,14 @@ function _load_run_table_df(study::String, run::String, table::String; group::Un db_path = joinpath(dir, "results.duckdb") isfile(db_path) || return (nothing, nothing, json_error(404, "no_results", "No results database for run '$run'")) - db = DuckDB.DB(db_path) - con = DBInterface.connect(db) - try + with_results_db(dir) do con cols = _duckdb_columns(con, table) isempty(cols) && return (nothing, nothing, json_error(404, "table_not_found", "Table '$table' not found in run '$run'")) scols = _sample_count_columns(con, table) order = _stable_order_clause(cols) df = DataFrame(DBInterface.execute(con, "SELECT * FROM \"$table\" $order")) - return (df, (db_path = db_path, sample_cols = scols, columns = cols), nothing) - finally - DBInterface.close!(con) - close(db) + (df, (db_path = db_path, sample_cols = scols, columns = cols), nothing) end end @@ -138,20 +133,20 @@ end dir = _require_duckdb(study, run; group) db_path = isnothing(dir) ? nothing : joinpath(dir, "results.duckdb") tables_info = OrderedDict{String,Any}[] - if !isnothing(db_path) && isfile(db_path) - db = DuckDB.DB(db_path) - con = DBInterface.connect(db) - try + if !isnothing(dir) && !isnothing(db_path) && isfile(db_path) + tables_info = with_results_db(dir) do con + available = OrderedDict{String,Any}[] names_df = DataFrame(DBInterface.execute(con, "SHOW TABLES")) tlist = sort(String[string(r.name) for r in eachrow(names_df)]) for t in tlist cols = _duckdb_columns(con, t) scols = _sample_count_columns(con, t) isempty(scols) && continue - nrows = Int(only(DataFrame(DBInterface.execute(con, "SELECT COUNT(*) AS n FROM \"$t\""))).n) + nrows = Int(only(DataFrame(DBInterface.execute(con, + "SELECT COUNT(*) AS n FROM \"$t\""))).n) has_seq = any(c -> c in ("Sequence", "sequence", "seq", "DNA"), cols) ranks = [r for r in MetaManifold.CladeCumulus.DEFAULT_TAXONOMIC_RANKS if r in cols] - push!(tables_info, OrderedDict{String,Any}( + push!(available, OrderedDict{String,Any}( "table" => t, "row_count" => nrows, "sample_count" => length(scols), @@ -159,9 +154,7 @@ end "ranks" => ranks, )) end - finally - DBInterface.close!(con) - close(db) + available end end diff --git a/test/unit/test_validated_statistics.jl b/test/unit/test_validated_statistics.jl index e4c64d70..74bf68b0 100644 --- a/test/unit/test_validated_statistics.jl +++ b/test/unit/test_validated_statistics.jl @@ -1,25 +1,25 @@ using Test using JSON3 using MetaManifold -SV = MetaManifold.Server +const VS_SERVER = MetaManifold.Server function _vs_request(method::String, path::String, body; raw::Bool=false) payload = raw ? body : JSON3.write(body) - req = SV.HTTP.Request(method, path, ["Content-Type" => "application/json"], payload) - SV.Oxygen.internalrequest(req) + req = VS_SERVER.HTTP.Request(method, path, ["Content-Type" => "application/json"], payload) + VS_SERVER.Oxygen.internalrequest(req) end _vs_post(path, body; raw=false) = _vs_request("POST", path, body; raw) _vs_get(path) = _vs_request("GET", path, "") function _vs_with_root(f::Function) root = mktempdir() - old = SV.ServerState._root[] + old = VS_SERVER.ServerState._root[] try - SV.ServerState.set_root!(root) + VS_SERVER.ServerState.set_root!(root) mkpath(joinpath(root, "projects")) f(root) finally - SV.ServerState._root[] = old + VS_SERVER.ServerState._root[] = old rm(root; recursive=true, force=true) end end From cb3fac0c9af9bd7606ac2132d5e461d9ea21d4ba Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:13:28 +0000 Subject: [PATCH 9/9] fix: count each active branch once in drag-and-drop phylogenetic diversity --- src/analysis/clade_cumulus.jl | 5 +++-- test/unit/test_clade_cumulus.jl | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/analysis/clade_cumulus.jl b/src/analysis/clade_cumulus.jl index 6c6d6f69..106f4869 100644 --- a/src/analysis/clade_cumulus.jl +++ b/src/analysis/clade_cumulus.jl @@ -1788,11 +1788,12 @@ function _reaggregate_after_move( mono = get(total_tips_per_taxon, u.dominant_taxon, 0) == 1 end active_here = u.cumulative_count > 0.0 + sub_pd = active_here && !is_root ? max(0.0, u.branch_length) : 0.0 st = Epistemic.epistemic_status(case_obj, u.id) leaf_node = CladeNode( u.id, u.name, u.rank, u.direct_count, u.cumulative_count, freq, u.residual_count, Epistemic.cloud_size_by_residual(u.residual_count), - u.branch_length, u.support, 0.0, u.dominant_taxon, u.taxon_purity, mono, + u.branch_length, u.support, sub_pd, u.dominant_taxon, u.taxon_purity, mono, st, Epistemic.epistemic_colour(st), CladeNode[], ) return (leaf_node, tmap, 1, active_here) @@ -1814,7 +1815,7 @@ function _reaggregate_after_move( n_sub_tips += nt if ch_active any_active = true - sub_pd += max(0.0, rk.branch_length) + rk.subtree_pd + sub_pd += rk.subtree_pd end for (k, v) in ct merged_taxa[k] = get(merged_taxa, k, 0) + v diff --git a/test/unit/test_clade_cumulus.jl b/test/unit/test_clade_cumulus.jl index 28499525..1e877abf 100644 --- a/test/unit/test_clade_cumulus.jl +++ b/test/unit/test_clade_cumulus.jl @@ -275,6 +275,31 @@ echo "(MMTIP_000001:0.05,(MMTIP_000002:0.03,MMTIP_000003:0.04)0.910:0.12,MMTIP_0 end end + @testset "Nested-clade drag-and-drop counts each active branch once" begin + df = DataFrame( + SeqName = ["ASV_1", "ASV_2", "ASV_3", "ASV_4", "ASV_5", "ASV_6"], + S1 = [10, 20, 30, 40, 50, 0], + ) + tree = CC.build_clade_tree(df; mode = :phylogenetic, newick = + "((ASV_1:0.10,ASV_2:0.20,ASV_6:9.0)0.96:0.30,(ASV_3:0.40,ASV_4:0.50,ASV_5:0.70)0.97:0.60)Root:8.0;") + source, target = tree.root.children + verdict = CC.validate_drag_drop(tree, source.id, target.id) + @test verdict["allowed"] + proposed = verdict["proposed_root"] + target_after = only(proposed["children"]) + source_after = last(target_after["children"]) + + # 0.10 + 0.20 + 0.30 + 0.40 + 0.50 + 0.70 + 0.60 = 2.80. + # The inactive tip's 9.0 and the root's 8.0 contribute nothing. + @test proposed["subtree_pd"] ≈ 2.80 atol = 1e-12 + @test target_after["subtree_pd"] ≈ 2.80 atol = 1e-12 + @test source_after["subtree_pd"] ≈ 0.60 atol = 1e-12 + @test [ch["subtree_pd"] for ch in source_after["children"]] ≈ [0.10, 0.20, 0.0] atol = 1e-12 + @test proposed["cumulative_count"] == tree.root.cumulative_count == 150.0 + @test tree.root.subtree_pd ≈ 2.80 atol = 1e-12 + @test length(tree.root.children) == 2 + end + @testset "Dual-hierarchy CladeCumulus, polyphyly detection, drag-and-drop, and HTTP routes" begin tmp = mktempdir() old_root = SV.ServerState._root[]