fix(cladistics): merge #223 with main and preserve unplaced placement mass

frontend/src/api/types.ts

@@ -968,6 +968,15 @@ export interface CladeCumulusRequest {
 export interface CladeCumulusDiversity {
   samples: string[]
   mapped_tips: number
+  /**
+   * Reads per sample that no tip on the tree carries — a query whose kept
+   * `.jplace` like-weight ratios do not reach 1 contributes only that share,
+   * and an ASV the tree does not cover contributes nothing. This mass remains
+   * in each sample's library-size denominator for weighted UniFrac but is
+   * excluded from tree-tip abundances. Absent for responses produced before
+   * this field existed.
+   */
+  unplaced_reads?: Record<string, number>
   faith_pd: Record<string, number>
   unweighted_unifrac: number[][]
   weighted_unifrac: number[][]

frontend/src/components/CladeCumulusPanel.tsx

@@ -33,6 +33,11 @@ export interface CladeCumulusPanelProps {
   runs: CladeCumulusRunOption[]
 }
 
+/**
+ * Explore a study's run trees, diversity and unplaced reads, with CSV export
+ * and move previews that can be saved as sidecar proposals. Move checks use
+ * the last computed inputs until the user recomputes the result.
+ */
 export function CladeCumulusPanel({ study, runs }: CladeCumulusPanelProps) {
   const toast = useToast()
   const reqCounter = useRef(0)
@@ -249,6 +254,17 @@ export function CladeCumulusPanel({ study, runs }: CladeCumulusPanelProps) {
     return div.normalized_weighted_unifrac
   }, [result, unifracKind])
 
+  // Only samples that actually lost mass are worth naming; a run with a
+  // complete placement file (or one produced before the field existed) reports
+  // nothing here.
+  const unplacedSamples = useMemo<[string, number][]>(() => {
+    const div = result?.diversity
+    if (!div?.unplaced_reads) return []
+    return div.samples
+      .map(s => [s, div.unplaced_reads?.[s] ?? 0] as [string, number])
+      .filter(([, n]) => n > 0)
+  }, [result])
+
   const exportMatrixCsv = () => {
     if (!result?.diversity || !activeMatrix) return
     const csv = formatDistanceMatrixCsv(result.diversity.samples, activeMatrix)
@@ -700,6 +716,23 @@ export function CladeCumulusPanel({ study, runs }: CladeCumulusPanelProps) {
                 ))}
               </div>
 
+              {/*
+                Reads no tip carries: like-weight mass a placement left on no
+                edge, plus ASVs the tree does not cover. They remain in each
+                sample's library-size denominator for weighted UniFrac but are
+                excluded from tree-tip abundances.
+              */}
+              {unplacedSamples.length > 0 && (
+                <p style={{ fontSize: '.75rem', margin: '0 0 12px', color: 'var(--color-fg-muted, #57606a)' }}>
+                  Unplaced reads (carried by no tip — like-weight mass left on no edge, plus
+                  ASVs the tree does not cover — included in each sample’s library-size
+                  denominator for weighted UniFrac but excluded from tree-tip abundances):{' '}
+                  {unplacedSamples
+                    .map(([s, n]) => `${s} ${n.toFixed(2)}`)
+                    .join(', ')}
+                </p>
+              )}
+
               {/* Pairwise UniFrac table */}
               {activeMatrix && (
                 <div style={{ overflowX: 'auto' }}>

scripts/check-julia-syntax.jl

@@ -20,6 +20,17 @@ for file in files
                 issue = ex
                 break
             end
+            # A bare `end` at the top level parses without error and then fails
+            # at lowering (`syntax: "end" at top level`), which is how a file
+            # that lost its `module Foo` opener reaches CI looking syntactically
+            # fine. An `end` balances exactly one opener, so top level cannot
+            # hold one: this is a missing or duplicated opener, and the file
+            # would not load.
+            if ex === :end
+                issue = "stray `end` at top level (a `module`/`function`/... opener " *
+                        "is missing or was duplicated in this file)"
+                break
+            end
             next_position > position ||
                 error("Julia parser made no progress at code unit $position")
             position = next_position
@@ -30,7 +41,7 @@ for file in files
     if !isnothing(issue)
         failed = true
         println(stderr, "Julia parse failed in $file:")
-        println(stderr, sprint(showerror, issue))
+        println(stderr, issue isa AbstractString ? issue : sprint(showerror, issue))
     end
 end
 failed && exit(1)

src/server/routes/clade_cumulus.jl

@@ -10,7 +10,7 @@
 # Canonical run DuckDB tables are read-only here; staged reassignment proposals
 # write only study-local sidecars under `projects/{study}/trees/.clade_cumulus/`.
 
-using DataFrames, OrderedCollections, SHA, Dates, JSON3
+using DataFrames, OrderedCollections, SHA, Dates, JSON3, DuckDB, DBInterface
 
 const _CC_SIDECAR_MAX_BYTES = 5 * 1024 * 1024
 
@@ -96,6 +96,13 @@ function _resolve_tree_inputs_from_request(
     body,
     source_digests::OrderedDict{String,String},
 )
+    # JSON3 hands request fields back as SubStrings and `strip` keeps them that
+    # way, but `_valid_tree_file` and `_tree_format` are String-typed -- an
+    # unnormalised `treeFile` reached them as a SubString and raised a
+    # MethodError, which the handler had no catch for, so every `study_tree`
+    # request answered 500. Normalise once at this boundary, as
+    # `_load_run_table_df` does above and as `cladistics.jl` does at its own
+    # call into `_valid_tree_file`. (`lowercase` already returns a String.)
     tree_source = lowercase(strip(string(get(body, :treeSource, "auto"))))
     tree_file = String(strip(string(get(body, :treeFile, ""))))
     inline_newick = String(strip(string(get(body, :newick, ""))))
@@ -270,19 +277,29 @@ end
             tid = String(strip(string(r[id_col])))
             !haskey(row_by_id, tid) && (row_by_id[tid] = ri)
         end
-        # Map split `.jplace` tip IDs back to `(base_query_name, share)` weights.
+        # Map split `.jplace` tip IDs back to the share of their query's reads
+        # that belongs on that edge. The share is the candidate's raw
+        # like-weight ratio, never a ratio renormalised over the retained
+        # candidates: a multi-candidate placement already partitions the mass
+        # (its ratios sum to 1 within `PLACEMENT_WEIGHT_SUM_TOL`), while a
+        # single-candidate placement accepted at `PLACEMENT_SINGLE_MIN_LWR`
+        # carries only the mass behind that one site. Dividing by the retained
+        # total would turn a lone 0.6 placement into a weight of 1.0 and push
+        # the 0.4 the tool left unplaced onto the reported edge, inflating that
+        # tip in weighted UniFrac and in Faith's PD. The shortfall is reported
+        # as `unplaced_reads` below instead.
         tip_weights = Dict{String,Float64}()
         if jplace_str !== nothing && !isempty(strip(jplace_str))
             jd_div = MetaManifold.CladeCumulus.parse_jplace(jplace_str)
             for q in jd_div.queries
                 q.status === :valid && !isempty(q.candidates) || continue
-                tw = sum((c.like_weight for c in q.candidates); init = 0.0)
-                tw > 0.0 || continue
                 multi = length(q.candidates) > 1
                 for (qname, _) in q.names
                     for (ci, cand) in enumerate(q.candidates)
                         tid = multi ? string(qname, "@edge", cand.edge_num, "#", ci) : qname
-                        tip_weights[tid] = cand.like_weight / tw
+                        # `parse_jplace` has already rejected any ratio outside
+                        # [0, 1], so this needs no further clamping.
+                        tip_weights[tid] = cand.like_weight
                     end
                 end
             end
@@ -305,6 +322,36 @@ end
                     mat[mi, ci] = ismissing(v) ? 0.0 : max(0.0, Float64(v)) * w
                 end
             end
+
+            # Each sample's library size over the whole table, and the part of
+            # it that no tip carries. Reads go missing from `mat` two ways: a
+            # query whose kept like-weight ratios do not reach 1 contributes
+            # only that share, and an ASV the tree does not cover contributes
+            # nothing at all. Both are the same thing to a diversity metric --
+            # mass that is not on the tree.
+            #
+            # `sample_totals` is then handed to the weighted metrics as their
+            # denominator. Without it they divide by the placed-only total, so
+            # a sample whose single mapped query was placed at LWR 0.6 comes
+            # back out at relative abundance 1.0 and the raw like-weight
+            # weighting above is undone at the last step. Faith's PD and
+            # unweighted UniFrac read presence only and are unaffected.
+            sample_totals = Float64[]
+            unplaced_reads = OrderedDict{String,Float64}()
+            for (ci, sc) in enumerate(scols)
+                tot = 0.0
+                for v in df[!, sc]
+                    ismissing(v) && continue
+                    tot += max(0.0, Float64(v))
+                end
+                placed = sum(@view mat[:, ci])
+                # A multi-candidate partition only has to sum to one within
+                # `PLACEMENT_WEIGHT_SUM_TOL`, so the placed mass can sit a hair
+                # above the table total. The denominator must never fall below
+                # what the tree actually carries.
+                push!(sample_totals, max(tot, placed))
+                unplaced_reads[sc] = max(0.0, tot - placed)
+            end
             pd_by_sample = OrderedDict{String,Float64}(
                 sc => MetaManifold.CladeCumulus.faith_pd(@view(mat[:, ci]), taxa_vec, cc_res.phylo_root)
                 for (ci, sc) in enumerate(scols)
@@ -313,14 +360,17 @@ end
                 mat, taxa_vec, scols, cc_res.phylo_root; metric = :unweighted
             )
             w_raw = MetaManifold.CladeCumulus.unifrac_matrix(
-                mat, taxa_vec, scols, cc_res.phylo_root; metric = :weighted, normalized = false
+                mat, taxa_vec, scols, cc_res.phylo_root; metric = :weighted, normalized = false,
+                totals = sample_totals
             )
             w_norm = MetaManifold.CladeCumulus.unifrac_matrix(
-                mat, taxa_vec, scols, cc_res.phylo_root; metric = :weighted, normalized = true
+                mat, taxa_vec, scols, cc_res.phylo_root; metric = :weighted, normalized = true,
+                totals = sample_totals
             )
             top_branches = if length(scols) >= 2
                 first_n = MetaManifold.CladeCumulus.branch_unifrac_contributions(
-                    @view(mat[:, 1]), @view(mat[:, 2]), taxa_vec, cc_res.phylo_root
+                    @view(mat[:, 1]), @view(mat[:, 2]), taxa_vec, cc_res.phylo_root;
+                    total_a = sample_totals[1], total_b = sample_totals[2]
                 )
                 first_n[1:min(length(first_n), 15)]
             else
@@ -329,6 +379,7 @@ end
             diversity_payload = OrderedDict{String,Any}(
                 "samples"                     => scols,
                 "mapped_tips"                 => length(taxa_vec),
+                "unplaced_reads"              => unplaced_reads,
                 "faith_pd"                    => pd_by_sample,
                 "unweighted_unifrac"          => [uw[i, :] for i in 1:size(uw, 1)],
                 "weighted_unifrac"            => [w_raw[i, :] for i in 1:size(w_raw, 1)],

src/server/server.jl

@@ -118,7 +118,6 @@ module Server
     include(joinpath(@__DIR__, "routes", "phylogeny.jl"))
     include(joinpath(@__DIR__, "routes", "cladistics.jl"))
     include(joinpath(@__DIR__, "routes", "clade_cumulus.jl"))
-    include(joinpath(@__DIR__, "routes", "cladistics.jl"))
 
     ## Bad request middleware
     # Helpers deep in a route (e.g. `_req_group`) throw this to reject a request.

test/runtests.jl

@@ -93,8 +93,15 @@ using MetaManifold.Categories, MetaManifold.CompositionLibrary
 ## Unit tests (always run)
 @testset "MetaManifold" begin
 
+    # Each file gets its own testset so that a throw at a file's top level -- outside
+    # any of that file's own testsets -- is reported against the file that caused it
+    # and does not abort the loop. Bare `include` here meant one such throw silently
+    # skipped every remaining unit file (and the occupancy probe below) while the
+    # summary still looked like a single stray error.
     for f in _selected_files()
-        include(joinpath(@__DIR__, "unit", f))
+        @testset "$f" begin
+            include(joinpath(@__DIR__, "unit", f))
+        end
     end
 
     ## Occupancy probe and the two-thread disagreement test (MMW-01), with the