From 3a45cf1c50b20e80cc010794e431ca919bfe54af Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 21:54:23 +0500 Subject: [PATCH 1/6] =?UTF-8?q?test(cli):=20red=20=E2=80=94=20a=20shared?= =?UTF-8?q?=20source=20is=20parsed=20once=20per=20corpus=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full-corpus load benchmark over the 9,909-chunk ingest measured 111.7s and 9,909 source imports for only 400 unique sources — each tab parsed ~25×. A per-session sha256→Score cache cut it to 4.5s / 400 imports with an identical result (9,909 loaded both ways). `load_corpus_material_with` injects the importer so the count is observable. The test writes two chunks that share one source and asserts it is imported once; the loader currently imports per chunk, so it fails 2 != 1. The cache is the GREEN. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- cli/src/generation_input.rs | 177 ++++++++++++++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 7 deletions(-) diff --git a/cli/src/generation_input.rs b/cli/src/generation_input.rs index 74f2ec61..e912c935 100644 --- a/cli/src/generation_input.rs +++ b/cli/src/generation_input.rs @@ -10,12 +10,14 @@ //! **Experimental, `#[doc(hidden)]`**: a stability-exempt seam for tooling, not //! a public library surface. +use std::collections::HashMap; use std::fs; use std::path::Path; use griff_core::corpus::{source_sha256, ChunkMeta}; use griff_core::generation_input::{corpus_material, prepare_chunk, LoadedChunk}; use griff_core::import; +use griff_core::score::Score; pub use griff_core::generation_input::{ bar_rhythms, generation_request_from_score, gesture_control_from_chunks, CorpusMaterial, @@ -33,6 +35,17 @@ pub use griff_core::generation_input::{ /// # Errors /// [`GenerationInputError::Corpus`] when `dir` cannot be read. pub fn load_corpus_material(dir: &Path) -> Result { + load_corpus_material_with(dir, |bytes| import::import_score_auto(bytes).ok()) +} + +/// The load loop with the source importer injected, so a test can count how +/// often a source is parsed. The corpus is ~25x redundant (≈400 sources behind +/// ≈9,900 chunks), so parsing each tab once instead of once per chunk is the +/// difference between a ~112 s and a ~4.5 s full load — measured, not guessed. +fn load_corpus_material_with( + dir: &Path, + mut import: impl FnMut(&[u8]) -> Option, +) -> Result { let entries = fs::read_dir(dir).map_err(|e| { GenerationInputError::Corpus(format!("cannot read corpus dir {}: {e}", dir.display())) })?; @@ -43,10 +56,11 @@ pub fn load_corpus_material(dir: &Path) -> Result = HashMap::new(); let mut loaded = Vec::new(); let mut skipped = Vec::new(); for name in record_names { - match load_chunk(dir, &name) { + match load_chunk(dir, &name, &mut cache, &mut import) { Some(chunk) => loaded.push(chunk), None => skipped.push(name), } @@ -54,11 +68,17 @@ pub fn load_corpus_material(dir: &Path) -> Result Option { +/// Reads one chunk record and prepares it through core, importing its source +/// tab (or reusing an already-parsed one from `cache`). `None` when the record +/// does not parse, the source is missing/unimportable/hash-mismatched, or the +/// prepared slice carries no sounding track — the caller reports it as skipped. +fn load_chunk( + dir: &Path, + record_name: &str, + cache: &mut HashMap, + import: &mut impl FnMut(&[u8]) -> Option, +) -> Option { + let _ = cache; let meta: ChunkMeta = serde_json::from_str(&fs::read_to_string(dir.join(record_name)).ok()?).ok()?; let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; @@ -70,6 +90,149 @@ fn load_chunk(dir: &Path, record_name: &str) -> Option { return None; } } - let source = import::import_score_auto(&bytes).ok()?; + let source = import(&bytes)?; prepare_chunk(meta, &source) } + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::unwrap_used)] + + use super::{import, load_corpus_material_with, source_sha256, Score}; + use griff_core::corpus::{ChunkId, ChunkMeta, SourceFormat, SourceRef}; + use griff_core::event::{NoteMarks, Pitch, Tempo, Ticks, TimeSignature, Tuning, Velocity}; + use griff_core::midi; + use griff_core::score::{ + AtomEvent, AtomNote, EventGroup, EventGroupKind, LossReport, MasterBar, RepeatMarker, + Track, Voice, + }; + use griff_core::slice::TickRange; + use std::cell::Cell; + use std::{env, fs, process}; + + fn note(start: u32, pitch: u8) -> AtomEvent { + AtomEvent::Note(AtomNote { + absolute_start: Ticks(start), + duration: Ticks(480), + pitch: Pitch::new(pitch).unwrap(), + velocity: Velocity::new(90).unwrap(), + marks: NoteMarks::empty(), + position: None, + }) + } + + fn two_bar_source() -> Score { + let master_bars = (0..2usize) + .map(|i| { + let start = u32::try_from(i).unwrap().saturating_mul(1920); + MasterBar { + index: i, + tick_range: TickRange::new(Ticks(start), Ticks(start.saturating_add(1920))) + .unwrap(), + time_signature: TimeSignature { + numerator: 4, + denominator: 4, + }, + tempo: Tempo::new(120.0).unwrap(), + repeat: RepeatMarker::default(), + } + }) + .collect(); + let atoms = [ + note(0, 40), + note(480, 43), + note(960, 45), + note(1440, 47), + note(1920, 50), + note(2400, 47), + note(2880, 45), + note(3360, 43), + ]; + Score { + ticks_per_quarter: 480, + master_bars, + tracks: vec![Track { + name: None, + channel: 0, + voices: vec![Voice { + id: 0, + event_groups: atoms + .into_iter() + .map(|a| EventGroup { + kind: EventGroupKind::Single, + atoms: vec![a], + technique_spans: Vec::new(), + }) + .collect(), + }], + tuning: Tuning::standard_e(), + }], + source_meta: None, + loss: LossReport::new(), + } + } + + fn chunk_meta(id: &str, sha: &str, bars: (u32, u32)) -> ChunkMeta { + ChunkMeta { + id: ChunkId(id.to_owned()), + title: String::new(), + source: SourceRef { + filename: "s.mid".to_owned(), + format: SourceFormat::Midi, + bar_range: Some(bars), + track_index: Some(0), + sha256: Some(sha.to_owned()), + }, + tempo_bpm: 120.0, + ticks_per_quarter: 480, + time_signature: (4, 4), + tuning: "standard_e".to_owned(), + tags: Vec::new(), + boundaries: Vec::new(), + techniques: Vec::new(), + quality_flags: Vec::new(), + reviewer: None, + structure: None, + gesture: None, + complexity: None, + duplicate: None, + style_cohort: None, + ensemble: None, + rights: None, + created_at: String::new(), + updated_at: String::new(), + } + } + + #[test] + fn one_source_is_parsed_once_for_all_its_chunks() { + let dir = env::temp_dir().join(format!("griff_loadcache_{}", process::id())); + fs::create_dir_all(&dir).unwrap(); + let bytes = midi::export_score(&two_bar_source()).unwrap(); + let sha = source_sha256(&bytes); + fs::write(dir.join("s.mid"), &bytes).unwrap(); + for (id, bars) in [("a", (0_u32, 0_u32)), ("b", (1, 1))] { + fs::write( + dir.join(format!("{id}.chunk.json")), + serde_json::to_string(&chunk_meta(id, &sha, bars)).unwrap(), + ) + .unwrap(); + } + + let imports = Cell::new(0_usize); + let material = load_corpus_material_with(&dir, |b| { + imports.set(imports.get().saturating_add(1)); + import::import_score_auto(b).ok() + }) + .expect("corpus loads"); + + assert_eq!(material.references.len(), 2, "both chunks load"); + assert_eq!( + imports.get(), + 1, + "the shared source is parsed once, not once per chunk" + ); + + fs::remove_dir_all(&dir).ok(); + } +} From 4b5520bec15856c53ad34d10caf2bb5ad9796d89 Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 21:55:32 +0500 Subject: [PATCH 2/6] =?UTF-8?q?fix(cli):=20green=20=E2=80=94=20cache=20par?= =?UTF-8?q?sed=20sources=20by=20content=20hash=20within=20a=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `load_chunk` keys the parsed `Score` by the source's sha256 (filename for pre-v9 records) in a per-load `HashMap`, so a tab shared by many chunks is parsed once. The count test passes (1 import for 2 chunks), and the existing correctness test is unchanged — `prepare_chunk` slices an immutable `&Score`, so reuse is bit-identical to per-chunk parsing. The hash check lives on the cache-miss path, so a mismatched source is still a strict load failure, never bypassed by reuse. Full-corpus effect (measured): 111.7s → 4.5s, 9,909 source imports → 400, same 9,909 chunks loaded. No persisted-schema change; the cockpit's OPFS loader is a separate path and unaffected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- cli/src/generation_input.rs | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/cli/src/generation_input.rs b/cli/src/generation_input.rs index e912c935..4a18af64 100644 --- a/cli/src/generation_input.rs +++ b/cli/src/generation_input.rs @@ -78,20 +78,31 @@ fn load_chunk( cache: &mut HashMap, import: &mut impl FnMut(&[u8]) -> Option, ) -> Option { - let _ = cache; let meta: ChunkMeta = serde_json::from_str(&fs::read_to_string(dir.join(record_name)).ok()?).ok()?; - let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; - // A filename is not an identity: when the record pins the source's hash - // (schema v9), a same-named but different file must not silently supply the - // notes. A mismatch is a load failure, reported like a missing source. - if let Some(expected) = &meta.source.sha256 { - if &source_sha256(&bytes) != expected { - return None; + // Key the parsed source by its content hash (v9) — falling back to the + // filename for pre-v9 records — so every chunk of one tab reuses a single + // parse. Determinism is unchanged: `prepare_chunk` slices an immutable + // `&Score`, so a shared parse yields exactly the per-chunk-parse result. + let key = meta + .source + .sha256 + .clone() + .unwrap_or_else(|| meta.source.filename.clone()); + if !cache.contains_key(&key) { + let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; + // A filename is not an identity: when the record pins the source's hash + // (schema v9), a same-named but different file must not silently supply + // the notes. A mismatch is a load failure — and a cache miss, so the + // check runs for every distinct expected hash, never bypassed by reuse. + if let Some(expected) = &meta.source.sha256 { + if &source_sha256(&bytes) != expected { + return None; + } } + cache.insert(key.clone(), import(&bytes)?); } - let source = import(&bytes)?; - prepare_chunk(meta, &source) + prepare_chunk(meta, cache.get(&key)?) } #[cfg(test)] From c7b9603b86c52153ba5cb920fe3bcfd8ba31f888 Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 22:20:52 +0500 Subject: [PATCH 3/6] =?UTF-8?q?test(core):=20red=20=E2=80=94=20dedup-clust?= =?UTF-8?q?er=20builder=20groups=20near-duplicates=20by=20link=20(3A)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curation step 3A. The ingest corpus is ~26% near-duplicate (2,476 of 9,909 chunks carry a `duplicate` link), and sending a curator through all 9,909 would mean re-reviewing almost-identical phrases — the most expensive resource in the system. `build_clusters` groups a corpus's chunks into near-duplicate clusters (a representative + its variants) purely from `ChunkMeta`: no Score, no import, no I/O. The contract, pinned by ten tests: a transitive `p0 <- p1 <- p2` chain is one cluster of three; independent roots are separate singletons; the result is independent of input order; every chunk lands in exactly one cluster with sizes summing to the count. A malformed link never silently becomes a singleton — it is a typed diagnostic: dangling, self, forward (not-earlier), cross-run (different ensemble), cycle, unresolvable provenance. No automatic reject, not even for an exact quote. The function is a stub returning empty, so the cluster and diagnostic tests fail; `split_phrase` (the id `_p` reader the schema documents) is real. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- core/src/curation.rs | 253 +++++++++++++++++++++++++++++++++++++++++++ core/src/lib.rs | 1 + 2 files changed, 254 insertions(+) create mode 100644 core/src/curation.rs diff --git a/core/src/curation.rs b/core/src/curation.rs new file mode 100644 index 00000000..9b5ae81d --- /dev/null +++ b/core/src/curation.rs @@ -0,0 +1,253 @@ +//! Dedup-cluster builder (curation step 3A). +//! +//! Groups a corpus's phrase chunks into near-duplicate **clusters** — a +//! representative plus the variants that quote it — purely from [`ChunkMeta`]. +//! No `Score`, no MIDI import, no I/O: this only reads the persisted +//! `duplicate` links and turns them into the unit a curator *navigates*. It is +//! deliberately not a graph framework; it resolves one link relation and +//! reports every malformed link, so corrupt provenance surfaces rather than +//! quietly masquerading as a clean singleton. +//! +//! A cluster is a unit of **navigation and comparison, never of automatic +//! fate**: no member is accepted or rejected here, not even an exact +//! (`quote_share == 1.0`) repeat. The decision stays per chunk, downstream. + +use std::collections::{HashMap, HashSet}; + +use crate::corpus::{ChunkId, ChunkMeta}; + +/// A near-duplicate cluster: the representative phrase and the variants linked +/// to it (transitively). `size == 1 + variants.len()`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CurationCluster { + /// Cluster key — the representative's id. + pub key: ChunkId, + /// The root phrase (no `duplicate` link of its own). + pub representative: ChunkId, + /// The near-duplicate variants, sorted by id (deterministic). + pub variants: Vec, + /// `1 + variants.len()`. + pub size: usize, +} + +/// A malformed `duplicate` link. Surfaced, never silently collapsed into a +/// singleton — a broken link is a provenance defect, not a lone phrase. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ClusterDiagnostic { + /// A chunk carries a `duplicate` link but its id has no parseable `_p` + /// phrase suffix, so the link cannot be resolved. + UnresolvableProvenance { chunk: ChunkId }, + /// `duplicate.of` names a phrase not present in the corpus. + DanglingLink { chunk: ChunkId, target: ChunkId }, + /// A chunk links to itself. + SelfLink { chunk: ChunkId }, + /// `duplicate.of` is not strictly earlier than the chunk's own phrase — the + /// "quotes an *earlier* phrase" rule is broken. + ForwardLink { + chunk: ChunkId, + of: usize, + own: usize, + }, + /// The link resolves to a chunk in a different split run (its ensemble + /// group/part differs), so the two are not comparable variants. + CrossRunLink { chunk: ChunkId, target: ChunkId }, + /// The link chain loops. + Cycle { chunk: ChunkId }, +} + +/// Builds the near-duplicate clusters of `chunks`, with a diagnostic for every +/// malformed link. +/// +/// On a clean corpus every chunk lands in exactly one cluster and the +/// diagnostics are empty; a chunk on a broken link is left out of the clusters +/// and reported instead. Deterministic: the result does not depend on the input +/// order. +#[must_use] +#[allow(clippy::too_many_lines)] // one cohesive pass: resolve links, roots, then group +pub fn build_clusters(chunks: &[ChunkMeta]) -> (Vec, Vec) { + let _ = chunks; + (Vec::new(), Vec::new()) +} + +/// Splits a phrase-chunk id into `(run_prefix, phrase_index)` at its trailing +/// `_p` — the suffix the corpus schema pairs with `duplicate.of`. `None` +/// when the id has no such suffix. +fn split_phrase(id: &str) -> Option<(&str, usize)> { + let at = id.rfind("_p")?; + let (prefix, rest) = id.split_at(at); + let index = rest.strip_prefix("_p")?.parse::().ok()?; + Some((prefix, index)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used)] + + use super::{build_clusters, split_phrase, ClusterDiagnostic}; + use crate::corpus::{ChunkId, ChunkMeta, EnsembleRef, SourceFormat, SourceRef}; + use crate::novelty::PhraseDuplicate; + + fn chunk(id: &str, group: &str, part: u32, dup: Option) -> ChunkMeta { + ChunkMeta { + id: ChunkId(id.to_owned()), + title: String::new(), + source: SourceRef { + filename: "s.gp".to_owned(), + format: SourceFormat::Gp, + bar_range: Some((0, 1)), + track_index: Some(part), + sha256: Some("hash".to_owned()), + }, + tempo_bpm: 120.0, + ticks_per_quarter: 480, + time_signature: (4, 4), + tuning: "standard_e".to_owned(), + tags: Vec::new(), + boundaries: Vec::new(), + techniques: Vec::new(), + quality_flags: Vec::new(), + reviewer: None, + structure: None, + gesture: None, + complexity: None, + duplicate: dup.map(|of| PhraseDuplicate { + of, + quote_share: 1.0, + }), + style_cohort: None, + ensemble: Some(EnsembleRef { + group_id: group.to_owned(), + part_index: part, + }), + rights: None, + created_at: String::new(), + updated_at: String::new(), + } + } + + /// `g_p0` <- `g_p1` <- `g_p2`: a transitive chain must be one cluster of 3. + fn transitive_run() -> Vec { + vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p1", "g", 0, Some(0)), + chunk("g_g0_p2", "g", 0, Some(1)), + ] + } + + #[test] + fn split_phrase_reads_the_trailing_index() { + assert_eq!(split_phrase("dgd_care_g1_p12"), Some(("dgd_care_g1", 12))); + assert_eq!(split_phrase("no_suffix"), None); + } + + #[test] + fn a_transitive_chain_is_one_cluster() { + let (clusters, diags) = build_clusters(&transitive_run()); + assert!(diags.is_empty(), "clean links produce no diagnostics"); + assert_eq!(clusters.len(), 1); + let c = clusters.first().expect("one cluster"); + assert_eq!(c.representative.0, "g_g0_p0", "the linkless root leads"); + assert_eq!(c.size, 3); + assert_eq!( + c.variants, + vec![ChunkId("g_g0_p1".to_owned()), ChunkId("g_g0_p2".to_owned())], + "both later phrases are variants, resolved transitively" + ); + } + + #[test] + fn independent_roots_are_separate_singletons() { + let chunks = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p1", "g", 0, None), + ]; + let (clusters, diags) = build_clusters(&chunks); + assert!(diags.is_empty()); + assert_eq!(clusters.len(), 2); + assert!(clusters + .iter() + .all(|c| c.size == 1 && c.variants.is_empty())); + } + + #[test] + fn the_result_is_independent_of_input_order() { + let mut reversed = transitive_run(); + reversed.reverse(); + assert_eq!(build_clusters(&transitive_run()), build_clusters(&reversed)); + } + + #[test] + fn a_dangling_link_is_a_diagnostic_not_a_singleton() { + // p2 quotes an *earlier* phrase 1, but no p1 exists in the corpus. + let chunks = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p2", "g", 0, Some(1)), + ]; + let (clusters, diags) = build_clusters(&chunks); + assert_eq!(clusters.len(), 1, "only the valid root clusters"); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::DanglingLink { .. }] + )); + } + + #[test] + fn a_self_link_is_a_diagnostic() { + let chunks = vec![chunk("g_g0_p0", "g", 0, Some(0))]; + let (_, diags) = build_clusters(&chunks); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::SelfLink { .. }] + )); + } + + #[test] + fn a_forward_link_is_a_diagnostic() { + // p0 claims to quote a *later* phrase 1 — violates "earlier only". + let chunks = vec![ + chunk("g_g0_p0", "g", 0, Some(1)), + chunk("g_g0_p1", "g", 0, None), + ]; + let (_, diags) = build_clusters(&chunks); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::ForwardLink { of: 1, own: 0, .. }] + )); + } + + #[test] + fn a_cycle_is_a_diagnostic() { + // Two chunks quoting each other (both forward/back) — a loop. + let chunks = vec![ + chunk("g_g0_p0", "g", 0, Some(1)), + chunk("g_g0_p1", "g", 0, Some(0)), + ]; + let (_, diags) = build_clusters(&chunks); + assert!( + !diags.is_empty(), + "a link cycle must be reported, never silently clustered" + ); + } + + #[test] + fn a_cross_run_link_is_a_diagnostic() { + // p1's id resolves a target whose ensemble group differs. + let mut a = chunk("g_g0_p0", "other", 0, None); + a.id = ChunkId("g_g0_p0".to_owned()); + let b = chunk("g_g0_p1", "g", 0, Some(0)); + let (_, diags) = build_clusters(&[a, b]); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::CrossRunLink { .. }] + )); + } + + #[test] + fn every_chunk_lands_in_exactly_one_cluster_on_clean_input() { + let chunks = transitive_run(); + let (clusters, diags) = build_clusters(&chunks); + assert!(diags.is_empty()); + let total: usize = clusters.iter().map(|c| c.size).sum(); + assert_eq!(total, chunks.len(), "cluster sizes sum to the chunk count"); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 26786c38..696af3c4 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -10,6 +10,7 @@ pub mod classify; pub mod closure; pub mod complement; pub mod corpus; +pub mod curation; pub mod dump; pub mod event; pub mod feature; From 6f402f7928c7c69707ffaa2a10996d1e1b1dd608 Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 22:22:13 +0500 Subject: [PATCH 4/6] =?UTF-8?q?feat(core):=20green=20=E2=80=94=20build=20d?= =?UTF-8?q?edup=20clusters=20from=20duplicate=20links=20(3A)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `build_clusters` resolves each chunk's `duplicate.of` (a phrase index) to the `_p`-suffixed sibling in the same run, follows the chain transitively to the linkless root, and groups every chunk under its root — the representative plus sorted variants. A malformed link is excluded and reported as a typed diagnostic (dangling / self / forward / cross-run / cycle / unresolvable), never folded into a silent singleton. `quote_share` rides along as link metadata and is never used to reclassify; nothing is auto-rejected. Deterministic: the output is sorted by id and independent of input order. Ten RED tests pass. Pure ChunkMeta in, clusters out — no Score, no import, no I/O, so a curation queue can be built without touching the source tabs. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- core/src/curation.rs | 133 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 2 deletions(-) diff --git a/core/src/curation.rs b/core/src/curation.rs index 9b5ae81d..9b7d23ae 100644 --- a/core/src/curation.rs +++ b/core/src/curation.rs @@ -65,8 +65,137 @@ pub enum ClusterDiagnostic { #[must_use] #[allow(clippy::too_many_lines)] // one cohesive pass: resolve links, roots, then group pub fn build_clusters(chunks: &[ChunkMeta]) -> (Vec, Vec) { - let _ = chunks; - (Vec::new(), Vec::new()) + let by_id: HashMap<&str, &ChunkMeta> = chunks.iter().map(|c| (c.id.0.as_str(), c)).collect(); + let mut diagnostics = Vec::new(); + let mut bad: HashSet = HashSet::new(); + // id -> None (root) or Some(parent id); absent means excluded by a diagnostic. + let mut parent: HashMap> = HashMap::new(); + + for c in chunks { + let id = c.id.0.clone(); + let Some(dup) = &c.duplicate else { + parent.insert(id, None); + continue; + }; + let Some((prefix, own)) = split_phrase(&id) else { + diagnostics.push(ClusterDiagnostic::UnresolvableProvenance { + chunk: c.id.clone(), + }); + bad.insert(id); + continue; + }; + if dup.of == own { + diagnostics.push(ClusterDiagnostic::SelfLink { + chunk: c.id.clone(), + }); + bad.insert(id); + continue; + } + if dup.of > own { + diagnostics.push(ClusterDiagnostic::ForwardLink { + chunk: c.id.clone(), + of: dup.of, + own, + }); + bad.insert(id); + continue; + } + let target = format!("{prefix}_p{}", dup.of); + let Some(target_meta) = by_id.get(target.as_str()) else { + diagnostics.push(ClusterDiagnostic::DanglingLink { + chunk: c.id.clone(), + target: ChunkId(target), + }); + bad.insert(id); + continue; + }; + if target_meta.ensemble != c.ensemble { + diagnostics.push(ClusterDiagnostic::CrossRunLink { + chunk: c.id.clone(), + target: ChunkId(target), + }); + bad.insert(id); + continue; + } + parent.insert(id, Some(target)); + } + + // Resolve each chunk to its root, detecting a cycle or a chain into an + // excluded chunk. The "earlier only" rule already forbids cycles, so this + // is a defensive net rather than an expected path. + let mut root_of: HashMap = HashMap::new(); + for c in chunks { + let start = c.id.0.clone(); + if bad.contains(&start) || root_of.contains_key(&start) { + continue; + } + let mut path: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut cur = start; + let root = loop { + if !seen.insert(cur.clone()) { + for node in &path { + diagnostics.push(ClusterDiagnostic::Cycle { + chunk: ChunkId(node.clone()), + }); + } + break None; + } + match parent.get(&cur) { + Some(None) => break Some(cur.clone()), + Some(Some(next)) => { + path.push(cur.clone()); + cur = next.clone(); + } + None => break None, + } + }; + match root { + Some(root_id) => { + for node in path { + root_of.insert(node, root_id.clone()); + } + root_of.insert(root_id.clone(), root_id); + } + None => { + for node in path { + bad.insert(node); + } + } + } + } + + let mut members: HashMap> = HashMap::new(); + for c in chunks { + let id = &c.id.0; + if bad.contains(id) { + continue; + } + if let Some(root) = root_of.get(id) { + members.entry(root.clone()).or_default().push(id.clone()); + } + } + + let mut clusters: Vec = members + .into_iter() + .map(|(root, mut ids)| { + ids.sort(); + let variants: Vec = ids + .into_iter() + .filter(|i| i != &root) + .map(ChunkId) + .collect(); + let size = variants.len().saturating_add(1); + CurationCluster { + key: ChunkId(root.clone()), + representative: ChunkId(root), + variants, + size, + } + }) + .collect(); + clusters.sort_by(|a, b| a.key.0.cmp(&b.key.0)); + (clusters, diagnostics) } /// Splits a phrase-chunk id into `(run_prefix, phrase_index)` at its trailing From 7bb7c7e92a54ece9f8ff7bc1b008ca1e2b66c2ec Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 22:38:39 +0500 Subject: [PATCH 5/6] =?UTF-8?q?test(core):=20red=20=E2=80=94=20cluster=20d?= =?UTF-8?q?iagnostics=20must=20be=20order-independent=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The determinism claim covered the clusters (sorted) but not the diagnostics, which came out in input order — a hole a clean 0-diagnostic corpus can never expose (zero errors sort themselves). A permuted broken corpus (two danglings and a self-link) makes `build_clusters(&broken) != build_clusters(&shuffled)`, so the test fails. Replaces the earlier "cycle" test, which asserted only `!diags.is_empty()` and never built a real cycle (the `of < own` rule makes one unreachable through the public builder). A genuine cycle-defense test comes with the GREEN, against the extracted resolution. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- core/src/curation.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/core/src/curation.rs b/core/src/curation.rs index 9b7d23ae..7db9c4b0 100644 --- a/core/src/curation.rs +++ b/core/src/curation.rs @@ -345,17 +345,26 @@ mod tests { } #[test] - fn a_cycle_is_a_diagnostic() { - // Two chunks quoting each other (both forward/back) — a loop. - let chunks = vec![ - chunk("g_g0_p0", "g", 0, Some(1)), - chunk("g_g0_p1", "g", 0, Some(0)), + fn diagnostics_are_order_independent() { + // Several distinct defects: two danglings and a self-link. The + // diagnostics must come out in the same order however the input is + // permuted — determinism can't stop at the clusters and leave the + // error list to the iteration order. + let broken = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p3", "g", 0, Some(1)), // dangling: no p1 + chunk("g_g0_p4", "g", 0, Some(2)), // dangling: no p2 + chunk("g_g0_p5", "g", 0, Some(5)), // self-link ]; - let (_, diags) = build_clusters(&chunks); - assert!( - !diags.is_empty(), - "a link cycle must be reported, never silently clustered" + let mut shuffled = broken.clone(); + shuffled.reverse(); + assert_eq!( + build_clusters(&broken), + build_clusters(&shuffled), + "clusters and diagnostics are both order-independent" ); + // And the defects are genuinely there (not an empty-vs-empty tie). + assert_eq!(build_clusters(&broken).1.len(), 3); } #[test] From 91412d3baa47eccbaadde3d4ae2844aa9a8d6084 Mon Sep 17 00:00:00 2001 From: PhysShell Date: Sat, 18 Jul 2026 22:42:39 +0500 Subject: [PATCH 6/6] =?UTF-8?q?fix(core):=20green=20=E2=80=94=20sort=20dia?= =?UTF-8?q?gnostics;=20test=20the=20cycle=20defense=20for=20real?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrective fixes to 3A, both from review: - **Deterministic diagnostics.** `build_clusters` now sorts the diagnostics (via derived `Ord` on `ClusterDiagnostic`, and `Ord` on `ChunkId`), so the error list no longer depends on input order — the determinism guarantee held only for the clusters before, a hole a clean 0-diagnostic corpus can't expose. The RED permutation-of-a-broken-corpus test passes. - **A real cycle test.** The root resolution is extracted into `resolve_roots`, and a test feeds it a hand-built loop `p0 -> p1 -> p2 -> p0` and asserts the exact `ClusterDiagnostic::Cycle` for every member. The public builder cannot make a cycle (the earlier-only rule strictly decreases the phrase index), so the old test asserted only `!diags.is_empty()` and never exercised the loop — the "all six variants tested" claim was the lie, not the builder. Full-corpus numbers are unchanged (0 diagnostics, so the sort is a no-op there): 7,405 clusters, 2,504 variants, 9,909 chunks. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NkqJUU6d1sW1RAfvyHrqVM --- core/src/corpus.rs | 2 +- core/src/curation.rs | 140 ++++++++++++++++++++++++++++++------------- 2 files changed, 99 insertions(+), 43 deletions(-) diff --git a/core/src/corpus.rs b/core/src/corpus.rs index 73817da9..02bec3c5 100644 --- a/core/src/corpus.rs +++ b/core/src/corpus.rs @@ -60,7 +60,7 @@ pub const SCHEMA_VERSION: u32 = 9; // ── identifiers ─────────────────────────────────────────────────────────────── /// Unique, stable identifier for a corpus chunk (e.g. `"dgd_001"`). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct ChunkId(pub String); /// Lowercase-hex SHA-256 of a source file's bytes — the content identity a diff --git a/core/src/curation.rs b/core/src/curation.rs index 7db9c4b0..8a3a04ad 100644 --- a/core/src/curation.rs +++ b/core/src/curation.rs @@ -32,7 +32,11 @@ pub struct CurationCluster { /// A malformed `duplicate` link. Surfaced, never silently collapsed into a /// singleton — a broken link is a provenance defect, not a lone phrase. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// `Ord` is derived so a caller can sort diagnostics into a stable, input-order- +/// independent list; the order (by variant, then chunk/field) has no meaning +/// beyond determinism. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum ClusterDiagnostic { /// A chunk carries a `duplicate` link but its id has no parseable `_p` /// phrase suffix, so the link cannot be resolved. @@ -120,22 +124,70 @@ pub fn build_clusters(chunks: &[ChunkMeta]) -> (Vec, Vec = HashMap::new(); + let (root_of, cycle_diagnostics) = resolve_roots(&parent, &mut bad); + diagnostics.extend(cycle_diagnostics); + + let mut members: HashMap> = HashMap::new(); for c in chunks { - let start = c.id.0.clone(); - if bad.contains(&start) || root_of.contains_key(&start) { + let id = &c.id.0; + if bad.contains(id) { + continue; + } + if let Some(root) = root_of.get(id) { + members.entry(root.clone()).or_default().push(id.clone()); + } + } + + let mut clusters: Vec = members + .into_iter() + .map(|(root, mut ids)| { + ids.sort(); + let variants: Vec = ids + .into_iter() + .filter(|i| i != &root) + .map(ChunkId) + .collect(); + let size = variants.len().saturating_add(1); + CurationCluster { + key: ChunkId(root.clone()), + representative: ChunkId(root), + variants, + size, + } + }) + .collect(); + clusters.sort_by(|a, b| a.key.0.cmp(&b.key.0)); + // Diagnostics are sorted too — determinism cannot stop at the clusters and + // leave the error list in iteration order. + diagnostics.sort(); + (clusters, diagnostics) +} + +/// Walks each linked chunk in `parent` to its root, reporting a `Cycle` +/// diagnostic (and marking the members `bad`) if a chain loops. Structural +/// validation already forbids cycles through the public builder — the +/// `duplicate.of < own` rule makes the phrase index strictly decrease along +/// every edge — so this is a defensive net, reachable only by a hand-built +/// malformed parent map. Output is independent of `parent`'s iteration order. +fn resolve_roots( + parent: &HashMap>, + bad: &mut HashSet, +) -> (HashMap, Vec) { + let mut root_of: HashMap = HashMap::new(); + let mut cycles: Vec = Vec::new(); + let mut starts: Vec<&String> = parent.keys().collect(); + starts.sort(); + for start in starts { + if bad.contains(start) || root_of.contains_key(start) { continue; } let mut path: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); - let mut cur = start; + let mut cur = start.clone(); let root = loop { if !seen.insert(cur.clone()) { for node in &path { - diagnostics.push(ClusterDiagnostic::Cycle { + cycles.push(ClusterDiagnostic::Cycle { chunk: ChunkId(node.clone()), }); } @@ -164,38 +216,7 @@ pub fn build_clusters(chunks: &[ChunkMeta]) -> (Vec, Vec> = HashMap::new(); - for c in chunks { - let id = &c.id.0; - if bad.contains(id) { - continue; - } - if let Some(root) = root_of.get(id) { - members.entry(root.clone()).or_default().push(id.clone()); - } - } - - let mut clusters: Vec = members - .into_iter() - .map(|(root, mut ids)| { - ids.sort(); - let variants: Vec = ids - .into_iter() - .filter(|i| i != &root) - .map(ChunkId) - .collect(); - let size = variants.len().saturating_add(1); - CurationCluster { - key: ChunkId(root.clone()), - representative: ChunkId(root), - variants, - size, - } - }) - .collect(); - clusters.sort_by(|a, b| a.key.0.cmp(&b.key.0)); - (clusters, diagnostics) + (root_of, cycles) } /// Splits a phrase-chunk id into `(run_prefix, phrase_index)` at its trailing @@ -212,9 +233,10 @@ fn split_phrase(id: &str) -> Option<(&str, usize)> { mod tests { #![allow(clippy::expect_used)] - use super::{build_clusters, split_phrase, ClusterDiagnostic}; + use super::{build_clusters, resolve_roots, split_phrase, ClusterDiagnostic}; use crate::corpus::{ChunkId, ChunkMeta, EnsembleRef, SourceFormat, SourceRef}; use crate::novelty::PhraseDuplicate; + use std::collections::{HashMap, HashSet}; fn chunk(id: &str, group: &str, part: u32, dup: Option) -> ChunkMeta { ChunkMeta { @@ -367,6 +389,40 @@ mod tests { assert_eq!(build_clusters(&broken).1.len(), 3); } + #[test] + fn resolve_roots_reports_an_exact_cycle_from_a_malformed_parent_map() { + // The public builder cannot produce a cycle (the earlier-only rule makes + // the phrase index strictly decrease), so the defensive net is tested at + // its own seam with a hand-built loop p0 -> p1 -> p2 -> p0. + let parent: HashMap> = [ + ("p0".to_owned(), Some("p1".to_owned())), + ("p1".to_owned(), Some("p2".to_owned())), + ("p2".to_owned(), Some("p0".to_owned())), + ] + .into_iter() + .collect(); + let mut bad = HashSet::new(); + let (root_of, diags) = resolve_roots(&parent, &mut bad); + + assert!(root_of.is_empty(), "no chunk in a cycle gets a root"); + assert!( + diags + .iter() + .all(|d| matches!(d, ClusterDiagnostic::Cycle { .. })), + "only Cycle diagnostics: {diags:?}" + ); + let mut cycled: Vec<&str> = diags + .iter() + .filter_map(|d| match d { + ClusterDiagnostic::Cycle { chunk } => Some(chunk.0.as_str()), + _ => None, + }) + .collect(); + cycled.sort_unstable(); + cycled.dedup(); + assert_eq!(cycled, ["p0", "p1", "p2"], "every cycle member is reported"); + } + #[test] fn a_cross_run_link_is_a_diagnostic() { // p1's id resolves a target whose ensemble group differs.