diff --git a/cli/src/generation_input.rs b/cli/src/generation_input.rs index 74f2ec61..4a18af64 100644 --- a/cli/src/generation_input.rs +++ b/cli/src/generation_input.rs @@ -10,12 +10,14 @@ //! **Experimental, `#[doc(hidden)]`**: a stability-exempt seam for tooling, not //! a public library surface. +use std::collections::HashMap; use std::fs; use std::path::Path; use griff_core::corpus::{source_sha256, ChunkMeta}; use griff_core::generation_input::{corpus_material, prepare_chunk, LoadedChunk}; use griff_core::import; +use griff_core::score::Score; pub use griff_core::generation_input::{ bar_rhythms, generation_request_from_score, gesture_control_from_chunks, CorpusMaterial, @@ -33,6 +35,17 @@ pub use griff_core::generation_input::{ /// # Errors /// [`GenerationInputError::Corpus`] when `dir` cannot be read. pub fn load_corpus_material(dir: &Path) -> Result { + load_corpus_material_with(dir, |bytes| import::import_score_auto(bytes).ok()) +} + +/// The load loop with the source importer injected, so a test can count how +/// often a source is parsed. The corpus is ~25x redundant (≈400 sources behind +/// ≈9,900 chunks), so parsing each tab once instead of once per chunk is the +/// difference between a ~112 s and a ~4.5 s full load — measured, not guessed. +fn load_corpus_material_with( + dir: &Path, + mut import: impl FnMut(&[u8]) -> Option, +) -> Result { let entries = fs::read_dir(dir).map_err(|e| { GenerationInputError::Corpus(format!("cannot read corpus dir {}: {e}", dir.display())) })?; @@ -43,10 +56,11 @@ pub fn load_corpus_material(dir: &Path) -> Result = HashMap::new(); let mut loaded = Vec::new(); let mut skipped = Vec::new(); for name in record_names { - match load_chunk(dir, &name) { + match load_chunk(dir, &name, &mut cache, &mut import) { Some(chunk) => loaded.push(chunk), None => skipped.push(name), } @@ -54,22 +68,182 @@ pub fn load_corpus_material(dir: &Path) -> Result Option { +/// Reads one chunk record and prepares it through core, importing its source +/// tab (or reusing an already-parsed one from `cache`). `None` when the record +/// does not parse, the source is missing/unimportable/hash-mismatched, or the +/// prepared slice carries no sounding track — the caller reports it as skipped. +fn load_chunk( + dir: &Path, + record_name: &str, + cache: &mut HashMap, + import: &mut impl FnMut(&[u8]) -> Option, +) -> Option { let meta: ChunkMeta = serde_json::from_str(&fs::read_to_string(dir.join(record_name)).ok()?).ok()?; - let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; - // A filename is not an identity: when the record pins the source's hash - // (schema v9), a same-named but different file must not silently supply the - // notes. A mismatch is a load failure, reported like a missing source. - if let Some(expected) = &meta.source.sha256 { - if &source_sha256(&bytes) != expected { - return None; + // Key the parsed source by its content hash (v9) — falling back to the + // filename for pre-v9 records — so every chunk of one tab reuses a single + // parse. Determinism is unchanged: `prepare_chunk` slices an immutable + // `&Score`, so a shared parse yields exactly the per-chunk-parse result. + let key = meta + .source + .sha256 + .clone() + .unwrap_or_else(|| meta.source.filename.clone()); + if !cache.contains_key(&key) { + let bytes = fs::read(dir.join(&meta.source.filename)).ok()?; + // A filename is not an identity: when the record pins the source's hash + // (schema v9), a same-named but different file must not silently supply + // the notes. A mismatch is a load failure — and a cache miss, so the + // check runs for every distinct expected hash, never bypassed by reuse. + if let Some(expected) = &meta.source.sha256 { + if &source_sha256(&bytes) != expected { + return None; + } + } + cache.insert(key.clone(), import(&bytes)?); + } + prepare_chunk(meta, cache.get(&key)?) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::unwrap_used)] + + use super::{import, load_corpus_material_with, source_sha256, Score}; + use griff_core::corpus::{ChunkId, ChunkMeta, SourceFormat, SourceRef}; + use griff_core::event::{NoteMarks, Pitch, Tempo, Ticks, TimeSignature, Tuning, Velocity}; + use griff_core::midi; + use griff_core::score::{ + AtomEvent, AtomNote, EventGroup, EventGroupKind, LossReport, MasterBar, RepeatMarker, + Track, Voice, + }; + use griff_core::slice::TickRange; + use std::cell::Cell; + use std::{env, fs, process}; + + fn note(start: u32, pitch: u8) -> AtomEvent { + AtomEvent::Note(AtomNote { + absolute_start: Ticks(start), + duration: Ticks(480), + pitch: Pitch::new(pitch).unwrap(), + velocity: Velocity::new(90).unwrap(), + marks: NoteMarks::empty(), + position: None, + }) + } + + fn two_bar_source() -> Score { + let master_bars = (0..2usize) + .map(|i| { + let start = u32::try_from(i).unwrap().saturating_mul(1920); + MasterBar { + index: i, + tick_range: TickRange::new(Ticks(start), Ticks(start.saturating_add(1920))) + .unwrap(), + time_signature: TimeSignature { + numerator: 4, + denominator: 4, + }, + tempo: Tempo::new(120.0).unwrap(), + repeat: RepeatMarker::default(), + } + }) + .collect(); + let atoms = [ + note(0, 40), + note(480, 43), + note(960, 45), + note(1440, 47), + note(1920, 50), + note(2400, 47), + note(2880, 45), + note(3360, 43), + ]; + Score { + ticks_per_quarter: 480, + master_bars, + tracks: vec![Track { + name: None, + channel: 0, + voices: vec![Voice { + id: 0, + event_groups: atoms + .into_iter() + .map(|a| EventGroup { + kind: EventGroupKind::Single, + atoms: vec![a], + technique_spans: Vec::new(), + }) + .collect(), + }], + tuning: Tuning::standard_e(), + }], + source_meta: None, + loss: LossReport::new(), } } - let source = import::import_score_auto(&bytes).ok()?; - prepare_chunk(meta, &source) + + fn chunk_meta(id: &str, sha: &str, bars: (u32, u32)) -> ChunkMeta { + ChunkMeta { + id: ChunkId(id.to_owned()), + title: String::new(), + source: SourceRef { + filename: "s.mid".to_owned(), + format: SourceFormat::Midi, + bar_range: Some(bars), + track_index: Some(0), + sha256: Some(sha.to_owned()), + }, + tempo_bpm: 120.0, + ticks_per_quarter: 480, + time_signature: (4, 4), + tuning: "standard_e".to_owned(), + tags: Vec::new(), + boundaries: Vec::new(), + techniques: Vec::new(), + quality_flags: Vec::new(), + reviewer: None, + structure: None, + gesture: None, + complexity: None, + duplicate: None, + style_cohort: None, + ensemble: None, + rights: None, + created_at: String::new(), + updated_at: String::new(), + } + } + + #[test] + fn one_source_is_parsed_once_for_all_its_chunks() { + let dir = env::temp_dir().join(format!("griff_loadcache_{}", process::id())); + fs::create_dir_all(&dir).unwrap(); + let bytes = midi::export_score(&two_bar_source()).unwrap(); + let sha = source_sha256(&bytes); + fs::write(dir.join("s.mid"), &bytes).unwrap(); + for (id, bars) in [("a", (0_u32, 0_u32)), ("b", (1, 1))] { + fs::write( + dir.join(format!("{id}.chunk.json")), + serde_json::to_string(&chunk_meta(id, &sha, bars)).unwrap(), + ) + .unwrap(); + } + + let imports = Cell::new(0_usize); + let material = load_corpus_material_with(&dir, |b| { + imports.set(imports.get().saturating_add(1)); + import::import_score_auto(b).ok() + }) + .expect("corpus loads"); + + assert_eq!(material.references.len(), 2, "both chunks load"); + assert_eq!( + imports.get(), + 1, + "the shared source is parsed once, not once per chunk" + ); + + fs::remove_dir_all(&dir).ok(); + } } diff --git a/core/src/corpus.rs b/core/src/corpus.rs index 73817da9..02bec3c5 100644 --- a/core/src/corpus.rs +++ b/core/src/corpus.rs @@ -60,7 +60,7 @@ pub const SCHEMA_VERSION: u32 = 9; // ── identifiers ─────────────────────────────────────────────────────────────── /// Unique, stable identifier for a corpus chunk (e.g. `"dgd_001"`). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct ChunkId(pub String); /// Lowercase-hex SHA-256 of a source file's bytes — the content identity a diff --git a/core/src/curation.rs b/core/src/curation.rs new file mode 100644 index 00000000..8a3a04ad --- /dev/null +++ b/core/src/curation.rs @@ -0,0 +1,447 @@ +//! Dedup-cluster builder (curation step 3A). +//! +//! Groups a corpus's phrase chunks into near-duplicate **clusters** — a +//! representative plus the variants that quote it — purely from [`ChunkMeta`]. +//! No `Score`, no MIDI import, no I/O: this only reads the persisted +//! `duplicate` links and turns them into the unit a curator *navigates*. It is +//! deliberately not a graph framework; it resolves one link relation and +//! reports every malformed link, so corrupt provenance surfaces rather than +//! quietly masquerading as a clean singleton. +//! +//! A cluster is a unit of **navigation and comparison, never of automatic +//! fate**: no member is accepted or rejected here, not even an exact +//! (`quote_share == 1.0`) repeat. The decision stays per chunk, downstream. + +use std::collections::{HashMap, HashSet}; + +use crate::corpus::{ChunkId, ChunkMeta}; + +/// A near-duplicate cluster: the representative phrase and the variants linked +/// to it (transitively). `size == 1 + variants.len()`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CurationCluster { + /// Cluster key — the representative's id. + pub key: ChunkId, + /// The root phrase (no `duplicate` link of its own). + pub representative: ChunkId, + /// The near-duplicate variants, sorted by id (deterministic). + pub variants: Vec, + /// `1 + variants.len()`. + pub size: usize, +} + +/// A malformed `duplicate` link. Surfaced, never silently collapsed into a +/// singleton — a broken link is a provenance defect, not a lone phrase. +/// +/// `Ord` is derived so a caller can sort diagnostics into a stable, input-order- +/// independent list; the order (by variant, then chunk/field) has no meaning +/// beyond determinism. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum ClusterDiagnostic { + /// A chunk carries a `duplicate` link but its id has no parseable `_p` + /// phrase suffix, so the link cannot be resolved. + UnresolvableProvenance { chunk: ChunkId }, + /// `duplicate.of` names a phrase not present in the corpus. + DanglingLink { chunk: ChunkId, target: ChunkId }, + /// A chunk links to itself. + SelfLink { chunk: ChunkId }, + /// `duplicate.of` is not strictly earlier than the chunk's own phrase — the + /// "quotes an *earlier* phrase" rule is broken. + ForwardLink { + chunk: ChunkId, + of: usize, + own: usize, + }, + /// The link resolves to a chunk in a different split run (its ensemble + /// group/part differs), so the two are not comparable variants. + CrossRunLink { chunk: ChunkId, target: ChunkId }, + /// The link chain loops. + Cycle { chunk: ChunkId }, +} + +/// Builds the near-duplicate clusters of `chunks`, with a diagnostic for every +/// malformed link. +/// +/// On a clean corpus every chunk lands in exactly one cluster and the +/// diagnostics are empty; a chunk on a broken link is left out of the clusters +/// and reported instead. Deterministic: the result does not depend on the input +/// order. +#[must_use] +#[allow(clippy::too_many_lines)] // one cohesive pass: resolve links, roots, then group +pub fn build_clusters(chunks: &[ChunkMeta]) -> (Vec, Vec) { + let by_id: HashMap<&str, &ChunkMeta> = chunks.iter().map(|c| (c.id.0.as_str(), c)).collect(); + let mut diagnostics = Vec::new(); + let mut bad: HashSet = HashSet::new(); + // id -> None (root) or Some(parent id); absent means excluded by a diagnostic. + let mut parent: HashMap> = HashMap::new(); + + for c in chunks { + let id = c.id.0.clone(); + let Some(dup) = &c.duplicate else { + parent.insert(id, None); + continue; + }; + let Some((prefix, own)) = split_phrase(&id) else { + diagnostics.push(ClusterDiagnostic::UnresolvableProvenance { + chunk: c.id.clone(), + }); + bad.insert(id); + continue; + }; + if dup.of == own { + diagnostics.push(ClusterDiagnostic::SelfLink { + chunk: c.id.clone(), + }); + bad.insert(id); + continue; + } + if dup.of > own { + diagnostics.push(ClusterDiagnostic::ForwardLink { + chunk: c.id.clone(), + of: dup.of, + own, + }); + bad.insert(id); + continue; + } + let target = format!("{prefix}_p{}", dup.of); + let Some(target_meta) = by_id.get(target.as_str()) else { + diagnostics.push(ClusterDiagnostic::DanglingLink { + chunk: c.id.clone(), + target: ChunkId(target), + }); + bad.insert(id); + continue; + }; + if target_meta.ensemble != c.ensemble { + diagnostics.push(ClusterDiagnostic::CrossRunLink { + chunk: c.id.clone(), + target: ChunkId(target), + }); + bad.insert(id); + continue; + } + parent.insert(id, Some(target)); + } + + let (root_of, cycle_diagnostics) = resolve_roots(&parent, &mut bad); + diagnostics.extend(cycle_diagnostics); + + let mut members: HashMap> = HashMap::new(); + for c in chunks { + let id = &c.id.0; + if bad.contains(id) { + continue; + } + if let Some(root) = root_of.get(id) { + members.entry(root.clone()).or_default().push(id.clone()); + } + } + + let mut clusters: Vec = members + .into_iter() + .map(|(root, mut ids)| { + ids.sort(); + let variants: Vec = ids + .into_iter() + .filter(|i| i != &root) + .map(ChunkId) + .collect(); + let size = variants.len().saturating_add(1); + CurationCluster { + key: ChunkId(root.clone()), + representative: ChunkId(root), + variants, + size, + } + }) + .collect(); + clusters.sort_by(|a, b| a.key.0.cmp(&b.key.0)); + // Diagnostics are sorted too — determinism cannot stop at the clusters and + // leave the error list in iteration order. + diagnostics.sort(); + (clusters, diagnostics) +} + +/// Walks each linked chunk in `parent` to its root, reporting a `Cycle` +/// diagnostic (and marking the members `bad`) if a chain loops. Structural +/// validation already forbids cycles through the public builder — the +/// `duplicate.of < own` rule makes the phrase index strictly decrease along +/// every edge — so this is a defensive net, reachable only by a hand-built +/// malformed parent map. Output is independent of `parent`'s iteration order. +fn resolve_roots( + parent: &HashMap>, + bad: &mut HashSet, +) -> (HashMap, Vec) { + let mut root_of: HashMap = HashMap::new(); + let mut cycles: Vec = Vec::new(); + let mut starts: Vec<&String> = parent.keys().collect(); + starts.sort(); + for start in starts { + if bad.contains(start) || root_of.contains_key(start) { + continue; + } + let mut path: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut cur = start.clone(); + let root = loop { + if !seen.insert(cur.clone()) { + for node in &path { + cycles.push(ClusterDiagnostic::Cycle { + chunk: ChunkId(node.clone()), + }); + } + break None; + } + match parent.get(&cur) { + Some(None) => break Some(cur.clone()), + Some(Some(next)) => { + path.push(cur.clone()); + cur = next.clone(); + } + None => break None, + } + }; + match root { + Some(root_id) => { + for node in path { + root_of.insert(node, root_id.clone()); + } + root_of.insert(root_id.clone(), root_id); + } + None => { + for node in path { + bad.insert(node); + } + } + } + } + (root_of, cycles) +} + +/// Splits a phrase-chunk id into `(run_prefix, phrase_index)` at its trailing +/// `_p` — the suffix the corpus schema pairs with `duplicate.of`. `None` +/// when the id has no such suffix. +fn split_phrase(id: &str) -> Option<(&str, usize)> { + let at = id.rfind("_p")?; + let (prefix, rest) = id.split_at(at); + let index = rest.strip_prefix("_p")?.parse::().ok()?; + Some((prefix, index)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used)] + + use super::{build_clusters, resolve_roots, split_phrase, ClusterDiagnostic}; + use crate::corpus::{ChunkId, ChunkMeta, EnsembleRef, SourceFormat, SourceRef}; + use crate::novelty::PhraseDuplicate; + use std::collections::{HashMap, HashSet}; + + fn chunk(id: &str, group: &str, part: u32, dup: Option) -> ChunkMeta { + ChunkMeta { + id: ChunkId(id.to_owned()), + title: String::new(), + source: SourceRef { + filename: "s.gp".to_owned(), + format: SourceFormat::Gp, + bar_range: Some((0, 1)), + track_index: Some(part), + sha256: Some("hash".to_owned()), + }, + tempo_bpm: 120.0, + ticks_per_quarter: 480, + time_signature: (4, 4), + tuning: "standard_e".to_owned(), + tags: Vec::new(), + boundaries: Vec::new(), + techniques: Vec::new(), + quality_flags: Vec::new(), + reviewer: None, + structure: None, + gesture: None, + complexity: None, + duplicate: dup.map(|of| PhraseDuplicate { + of, + quote_share: 1.0, + }), + style_cohort: None, + ensemble: Some(EnsembleRef { + group_id: group.to_owned(), + part_index: part, + }), + rights: None, + created_at: String::new(), + updated_at: String::new(), + } + } + + /// `g_p0` <- `g_p1` <- `g_p2`: a transitive chain must be one cluster of 3. + fn transitive_run() -> Vec { + vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p1", "g", 0, Some(0)), + chunk("g_g0_p2", "g", 0, Some(1)), + ] + } + + #[test] + fn split_phrase_reads_the_trailing_index() { + assert_eq!(split_phrase("dgd_care_g1_p12"), Some(("dgd_care_g1", 12))); + assert_eq!(split_phrase("no_suffix"), None); + } + + #[test] + fn a_transitive_chain_is_one_cluster() { + let (clusters, diags) = build_clusters(&transitive_run()); + assert!(diags.is_empty(), "clean links produce no diagnostics"); + assert_eq!(clusters.len(), 1); + let c = clusters.first().expect("one cluster"); + assert_eq!(c.representative.0, "g_g0_p0", "the linkless root leads"); + assert_eq!(c.size, 3); + assert_eq!( + c.variants, + vec![ChunkId("g_g0_p1".to_owned()), ChunkId("g_g0_p2".to_owned())], + "both later phrases are variants, resolved transitively" + ); + } + + #[test] + fn independent_roots_are_separate_singletons() { + let chunks = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p1", "g", 0, None), + ]; + let (clusters, diags) = build_clusters(&chunks); + assert!(diags.is_empty()); + assert_eq!(clusters.len(), 2); + assert!(clusters + .iter() + .all(|c| c.size == 1 && c.variants.is_empty())); + } + + #[test] + fn the_result_is_independent_of_input_order() { + let mut reversed = transitive_run(); + reversed.reverse(); + assert_eq!(build_clusters(&transitive_run()), build_clusters(&reversed)); + } + + #[test] + fn a_dangling_link_is_a_diagnostic_not_a_singleton() { + // p2 quotes an *earlier* phrase 1, but no p1 exists in the corpus. + let chunks = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p2", "g", 0, Some(1)), + ]; + let (clusters, diags) = build_clusters(&chunks); + assert_eq!(clusters.len(), 1, "only the valid root clusters"); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::DanglingLink { .. }] + )); + } + + #[test] + fn a_self_link_is_a_diagnostic() { + let chunks = vec![chunk("g_g0_p0", "g", 0, Some(0))]; + let (_, diags) = build_clusters(&chunks); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::SelfLink { .. }] + )); + } + + #[test] + fn a_forward_link_is_a_diagnostic() { + // p0 claims to quote a *later* phrase 1 — violates "earlier only". + let chunks = vec![ + chunk("g_g0_p0", "g", 0, Some(1)), + chunk("g_g0_p1", "g", 0, None), + ]; + let (_, diags) = build_clusters(&chunks); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::ForwardLink { of: 1, own: 0, .. }] + )); + } + + #[test] + fn diagnostics_are_order_independent() { + // Several distinct defects: two danglings and a self-link. The + // diagnostics must come out in the same order however the input is + // permuted — determinism can't stop at the clusters and leave the + // error list to the iteration order. + let broken = vec![ + chunk("g_g0_p0", "g", 0, None), + chunk("g_g0_p3", "g", 0, Some(1)), // dangling: no p1 + chunk("g_g0_p4", "g", 0, Some(2)), // dangling: no p2 + chunk("g_g0_p5", "g", 0, Some(5)), // self-link + ]; + let mut shuffled = broken.clone(); + shuffled.reverse(); + assert_eq!( + build_clusters(&broken), + build_clusters(&shuffled), + "clusters and diagnostics are both order-independent" + ); + // And the defects are genuinely there (not an empty-vs-empty tie). + assert_eq!(build_clusters(&broken).1.len(), 3); + } + + #[test] + fn resolve_roots_reports_an_exact_cycle_from_a_malformed_parent_map() { + // The public builder cannot produce a cycle (the earlier-only rule makes + // the phrase index strictly decrease), so the defensive net is tested at + // its own seam with a hand-built loop p0 -> p1 -> p2 -> p0. + let parent: HashMap> = [ + ("p0".to_owned(), Some("p1".to_owned())), + ("p1".to_owned(), Some("p2".to_owned())), + ("p2".to_owned(), Some("p0".to_owned())), + ] + .into_iter() + .collect(); + let mut bad = HashSet::new(); + let (root_of, diags) = resolve_roots(&parent, &mut bad); + + assert!(root_of.is_empty(), "no chunk in a cycle gets a root"); + assert!( + diags + .iter() + .all(|d| matches!(d, ClusterDiagnostic::Cycle { .. })), + "only Cycle diagnostics: {diags:?}" + ); + let mut cycled: Vec<&str> = diags + .iter() + .filter_map(|d| match d { + ClusterDiagnostic::Cycle { chunk } => Some(chunk.0.as_str()), + _ => None, + }) + .collect(); + cycled.sort_unstable(); + cycled.dedup(); + assert_eq!(cycled, ["p0", "p1", "p2"], "every cycle member is reported"); + } + + #[test] + fn a_cross_run_link_is_a_diagnostic() { + // p1's id resolves a target whose ensemble group differs. + let mut a = chunk("g_g0_p0", "other", 0, None); + a.id = ChunkId("g_g0_p0".to_owned()); + let b = chunk("g_g0_p1", "g", 0, Some(0)); + let (_, diags) = build_clusters(&[a, b]); + assert!(matches!( + diags.as_slice(), + [ClusterDiagnostic::CrossRunLink { .. }] + )); + } + + #[test] + fn every_chunk_lands_in_exactly_one_cluster_on_clean_input() { + let chunks = transitive_run(); + let (clusters, diags) = build_clusters(&chunks); + assert!(diags.is_empty()); + let total: usize = clusters.iter().map(|c| c.size).sum(); + assert_eq!(total, chunks.len(), "cluster sizes sum to the chunk count"); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 26786c38..696af3c4 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -10,6 +10,7 @@ pub mod classify; pub mod closure; pub mod complement; pub mod corpus; +pub mod curation; pub mod dump; pub mod event; pub mod feature;