From e7d34eaf76ea8be79a55b479e70a4ef5d3a555b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:33 +0000 Subject: [PATCH 01/19] Harvest C++ method bodies into the recipe fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ruff_cpp_spo`'s body arm existed as an unwired draft: the fingerprint the fuzzy recipe codebook classifies on (`writes`/`reads`/`raises`/`calls` plus the J1 `guarded_writes` guard) was computed by `method_body_arm` and then thrown away, because nothing on `CppMethod` could carry it and no expand arm could emit it. C++ was the one frontend whose bodies produced no facts. Wire it end to end, and correct the draft against measured libclang shapes. The draft had three defects, each found by dumping real cursors rather than reading the AST docs: * libclang exposes no binary-operator kind, so the draft recorded the LHS of ANY binary operator as a write — `if (status_ == v)` reported a mutation. The operator is now read off the token stream. * the assignment target also landed in `reads`, so a pure setter classified as `Normalize` instead of `Compute`. * `guarded_writes` was never populated, so the J1 split the codebook needs could not fire on C++ at all. Two shapes the draft could not have known without measuring: a class-typed member assigns through `operator=`, so it arrives as a `CallExpr` and not a `BinaryOperator`; and a call's own callee reference is a `MemberRefExpr` that would otherwise be recorded as a read of a field named after the method. Also: * `walk_tu` now parses bodies; `walk_tu_configured` opts out for a signature-only walk, and takes a `BodyArmConfig` so a corpus can supply its own mutator vocabulary instead of the shipped closed set. * the arm is harvested from `get_definition()`, and `walk_files` merges arms across translation units — without both, every out-of-line definition (most of the behaviour in a real C++ corpus) harvests nothing. * `recipe::classify` is generic over a new `BodyFacts` trait implemented for both `Function` and `CppMethod`. Its docs already claimed it classified C++ methods; its signature accepted only `Function`, so it could not. * `CppMethod` joins the DTO-surface lock as `CPP_METHOD_DTO`, pinned against `FUNCTION_DTO` so the two frontends cannot drift apart silently. Every guard is disable-verified red-then-green (16 checks). Two of those runs came back green and were investigated rather than accepted: one was a mis-aimed disable, the other a genuinely vacuous fixture — the header in the cross-TU test parsed as C, not C++, so the two-sighting condition it measures never occurred and the assertions held with the merge removed. That fixture now passes `-x c++` and asserts up front that the header's own sighting is arm-less. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/Cargo.toml | 6 +- .../ruff_cpp_spo/examples/harvest_ladybug.rs | 281 +++++ crates/ruff_cpp_spo/src/clang_walker.rs | 1078 +++++++++++++---- crates/ruff_cpp_spo/src/lib.rs | 161 ++- crates/ruff_spo_triplet/src/expand.rs | 152 +++ crates/ruff_spo_triplet/src/ir.rs | 128 +- crates/ruff_spo_triplet/src/lib.rs | 2 +- crates/ruff_spo_triplet/src/reassemble.rs | 161 +++ crates/ruff_spo_triplet/src/recipe.rs | 201 ++- 9 files changed, 1915 insertions(+), 255 deletions(-) create mode 100644 crates/ruff_cpp_spo/examples/harvest_ladybug.rs diff --git a/crates/ruff_cpp_spo/Cargo.toml b/crates/ruff_cpp_spo/Cargo.toml index 93aa184ee7380a..32ef2b336ffac9 100644 --- a/crates/ruff_cpp_spo/Cargo.toml +++ b/crates/ruff_cpp_spo/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" publish = false edition = "2024" rust-version = "1.86" -description = "C++ machine-plane frontend for ruff_spo_triplet. Walks a C++ corpus (Tesseract first) via libclang and fills a language-agnostic ModelGraph, which expands to the same SPO triples as the Python/Odoo and Ruby/Rails frontends. The per-TU walker (`walk_tu`, behind the `libclang` feature) is implemented and tested against real libclang; the corpus-tree orchestration (`extract`) remains. C++ source corpora stay UPSTREAM — never vendored into this repo or into tesseract-rs." +description = "C++ machine-plane frontend for ruff_spo_triplet. Walks a C++ corpus (Tesseract first) via libclang and fills a language-agnostic ModelGraph, which expands to the same SPO triples as the Python/Odoo and Ruby/Rails frontends. The per-TU walker (`walk_tu`, behind the `libclang` feature) is implemented and tested against real libclang, as are the corpus-tree walkers (`extract_dir`, `extract_tree`); only per-TU include-arg auto-detection (`extract`) remains. C++ source corpora stay UPSTREAM — never vendored into this repo or into tesseract-rs." license = "MIT" [lib] @@ -53,6 +53,10 @@ required-features = ["libclang"] name = "harvest_textord" required-features = ["libclang"] +[[example]] +name = "harvest_ladybug" +required-features = ["libclang"] + [dev-dependencies] # End-to-end pipeline test only (harvest -> reassemble -> ruff_cpp_codegen emit). # The harvester does NOT depend on the codegen at runtime — this is the test diff --git a/crates/ruff_cpp_spo/examples/harvest_ladybug.rs b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs new file mode 100644 index 00000000000000..4fbdce6c706386 --- /dev/null +++ b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs @@ -0,0 +1,281 @@ +//! Harvest an arbitrary ladybug (KuzuDB-fork) header **subtree**'s C++ class +//! manifest via libclang, recursively, and emit the SPO manifest plus a full +//! breakdown (`inherits_from` trees, `has_function` top-25, pure-virtual +//! classes, `virtually_overrides` count, and field, template, and +//! static-assert counts) — the `ruff>OGAR` structure feeding the ladybug-rs +//! transcode. +//! +//! Walks every `*.h` header under `src/include/` (recursively) via +//! [`walk_tu_with_diagnostics`] (not [`walk_tu`](ruff_cpp_spo::walk_tu)) and +//! warns loudly when a header's diagnostic list is non-empty. Per that +//! function's own docs: "0 failed" from `walk_tu` alone does NOT mean the +//! sweep is complete — libclang can recover from an unresolved `#include` by +//! treating the rest of the file as best-effort, silently dropping the +//! declaration that needed the missing header (see `examples/harvest_textord.rs`'s +//! `src/viewer` note for the concrete failure mode this was found against). +//! +//! Env: +//! `LADYBUG_SRC` default /home/user/ladybug +//! `SUBTREE` relative to src/include, e.g. "storage" or "processor" (default "storage") +//! `MANIFEST_OUT` ndjson output path (default /tmp/ladybug__manifest.ndjson) +//! +//! Run: +//! ```sh +//! LADYBUG_SRC=/home/user/ladybug LIBCLANG_PATH=/usr/lib/llvm-18/lib SUBTREE=storage \ +//! cargo run -p ruff_cpp_spo --features libclang --example harvest_ladybug +//! ``` + +#![expect( + clippy::print_stderr, + reason = "manifest-emission CLI example (mirrors harvest_textord)" +)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use ruff_cpp_spo::{CppClass, Declaration, NAMESPACE, model_from_class, walk_tu_with_diagnostics}; +use ruff_spo_triplet::{ModelGraph, expand, to_ndjson}; + +fn collect_headers(dir: &Path, out: &mut Vec) -> std::io::Result<()> { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_headers(&path, out)?; + } else if path.extension().is_some_and(|x| x == "h") { + out.push(path); + } + } + Ok(()) +} + +fn main() -> Result<(), Box> { + let root = std::env::var("LADYBUG_SRC").unwrap_or_else(|_| "/home/user/ladybug".to_string()); + let root = Path::new(&root); + let subtree = std::env::var("SUBTREE").unwrap_or_else(|_| "storage".to_string()); + let sub_dir = root.join("src/include").join(&subtree); + if !sub_dir.exists() { + return Err(format!("{} not found; set LADYBUG_SRC/SUBTREE", sub_dir.display()).into()); + } + + let args = [ + "-std=c++20".to_string(), + "-x".to_string(), + "c++".to_string(), + format!("-I{}", root.join("src/include").display()), + ]; + + let mut headers: Vec = Vec::new(); + collect_headers(&sub_dir, &mut headers)?; + headers.sort(); + + let mut all: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + let mut failed: Vec<(String, String)> = Vec::new(); + // Headers where the parse itself succeeded ("0 failed") but libclang + // still reported severity>=Error diagnostics — the silent-drop signature + // documented on `walk_tu_with_diagnostics`. Tracked separately from + // `failed` because a diagnostic-bearing header still yields a (possibly + // partial) class list here, unlike a hard `WalkError`. + let mut warned: Vec<(String, usize)> = Vec::new(); + for h in &headers { + let name = h.strip_prefix(&sub_dir).unwrap_or(h).display().to_string(); + match walk_tu_with_diagnostics(h, &args) { + Ok((classes, diagnostics)) => { + for c in classes { + if seen.insert(c.qualified_name()) { + all.push(c); + } + } + if !diagnostics.is_empty() { + eprintln!( + "[harvest:{subtree}] WARNING: {} unresolved-include error(s) in {name} — class list may be incomplete", + diagnostics.len(), + ); + for d in diagnostics.iter().take(5) { + eprintln!(" {d}"); + } + warned.push((name, diagnostics.len())); + } + } + Err(e) => failed.push((name, e.to_string())), + } + } + eprintln!( + "[harvest:{subtree}] {} headers walked, {} unique classes, {} failed, {} with unresolved-include diagnostics", + headers.len(), + all.len(), + failed.len(), + warned.len(), + ); + for (name, err) in failed.iter().take(20) { + eprintln!("[harvest:{subtree}] walk failed in {name}: {err}"); + } + if failed.len() > 20 { + eprintln!( + "[harvest:{subtree}] ... {} more failures omitted", + failed.len() - 20 + ); + } + if !warned.is_empty() { + eprintln!( + "\n[harvest:{subtree}] headers with unresolved-include diagnostics (class list may be incomplete):" + ); + for (name, n) in &warned { + eprintln!(" {name}: {n} error(s)"); + } + } + + // ── inherits_from edges, grouped by base (root -> subclasses) ── + eprintln!("\n[{subtree}] inherits_from edges (derived : base):"); + let mut base_to_subs: BTreeMap> = BTreeMap::new(); + for c in &all { + let bases: Vec<&str> = c + .declarations + .iter() + .filter_map(|d| match d { + Declaration::Base(b) => Some(b.name.as_str()), + _ => None, + }) + .collect(); + if !bases.is_empty() { + eprintln!(" {} : {}", c.qualified_name(), bases.join(", ")); + for b in bases { + base_to_subs + .entry(b.to_string()) + .or_default() + .push(c.qualified_name()); + } + } + } + + eprintln!("\n[{subtree}] base -> subclass counts (sorted by count desc):"); + let mut base_counts: Vec<(&String, &Vec)> = base_to_subs.iter().collect(); + base_counts.sort_by_key(|a| std::cmp::Reverse(a.1.len())); + for (base, subs) in &base_counts { + eprintln!(" {base}: {} subclasses", subs.len()); + } + + // ── has_function top-25 ── + eprintln!("\n[{subtree}] has_function top-25 by method count:"); + let mut counts: Vec<(String, usize, usize)> = all + .iter() + .map(|c| { + let methods: Vec<&_> = c + .declarations + .iter() + .filter_map(|d| match d { + Declaration::Method(m) => Some(m), + _ => None, + }) + .collect(); + let overrides = methods.iter().filter(|m| m.overrides.is_some()).count(); + (c.qualified_name(), methods.len(), overrides) + }) + .collect(); + counts.sort_by_key(|c| std::cmp::Reverse(c.1)); + for (name, n, ov) in counts.iter().take(25) { + eprintln!(" {name}: {n} methods, {ov} overrides"); + } + + // ── is_pure_virtual classes, ranked by pure-virtual method count ── + eprintln!("\n[{subtree}] pure-virtual-heavy classes (top 20):"); + let mut pv: Vec<(String, usize)> = all + .iter() + .map(|c| { + let n = c + .declarations + .iter() + .filter_map(|d| match d { + Declaration::Method(m) => Some(m), + _ => None, + }) + .filter(|m| m.is_pure_virtual) + .count(); + (c.qualified_name(), n) + }) + .filter(|(_, n)| *n > 0) + .collect(); + pv.sort_by_key(|p| std::cmp::Reverse(p.1)); + for (name, n) in pv.iter().take(20) { + eprintln!(" {name}: {n} pure-virtual methods"); + } + + // ── virtually_overrides count ── + let virt_overrides: usize = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter_map(|d| match d { + Declaration::Method(m) => Some(m), + _ => None, + }) + .filter(|m| m.overrides.is_some()) + .count(); + eprintln!("\n[{subtree}] total virtually_overrides edges: {virt_overrides}"); + + // ── fields / templates / static_asserts / friends ── + let n_fields: usize = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter(|d| matches!(d, Declaration::Field(_))) + .count(); + let n_templates: usize = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter(|d| matches!(d, Declaration::Template(_))) + .count(); + let n_static_asserts: usize = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter(|d| matches!(d, Declaration::StaticAssert(_))) + .count(); + let n_friends: usize = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter(|d| matches!(d, Declaration::Friend(_))) + .count(); + eprintln!( + "\n[{subtree}] has_field={n_fields} template_specialises/instantiates={n_templates} static_asserts={n_static_asserts} is_friend_of={n_friends}" + ); + + // top-10 classes by template decl count (heavy specialisation signal) + let mut tmpl_counts: Vec<(String, usize)> = all + .iter() + .map(|c| { + let n = c + .declarations + .iter() + .filter(|d| matches!(d, Declaration::Template(_))) + .count(); + (c.qualified_name(), n) + }) + .filter(|(_, n)| *n > 0) + .collect(); + tmpl_counts.sort_by_key(|t| std::cmp::Reverse(t.1)); + if !tmpl_counts.is_empty() { + eprintln!("\n[{subtree}] top classes by template decl count:"); + for (name, n) in tmpl_counts.iter().take(10) { + eprintln!(" {name}: {n} template decls"); + } + } + + // Emit the full ndjson manifest — what lance-graph's SPO store + the + // ladybug-rs codegen consume. + let mut graph = ModelGraph::new(NAMESPACE); + for c in &all { + graph.models.push(model_from_class(c)); + } + let triples = expand(&graph); + let ndjson = to_ndjson(&triples); + let out_path = std::env::var("MANIFEST_OUT") + .unwrap_or_else(|_| format!("/tmp/ladybug_{subtree}_manifest.ndjson")); + std::fs::write(&out_path, &ndjson)?; + eprintln!( + "\n[harvest:{subtree}] {} models -> {} triples, {} ndjson bytes -> {out_path}", + graph.models.len(), + triples.len(), + ndjson.len() + ); + + Ok(()) +} diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index ec59344f0edca5..d3039fa1d24452 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -14,6 +14,12 @@ //! `virtually_overrides`, `defines_operator`, `is_pure_virtual`, and //! `is_noexcept` predicates from real parsing. //! +//! It also harvests each method's **body arm** — the recipe fingerprint +//! (`writes_field` / `reads_field` / `raises` / `calls` / `writes_if_blank`) +//! the fuzzy recipe codebook classifies on, in the same shape the Ruby and C# +//! frontends produce. See the BODY ARM section below for the measured cursor +//! shapes it matches, and [`walk_tu_configured`] to opt out of it. +//! //! **Walker follow-ups** (the IR + predicates already exist from PR #8; only //! the walker does not populate them yet): `constexpr`/`consteval` and //! C++20 `requires` clauses (not surfaced by the high-level `clang` API — @@ -66,10 +72,12 @@ impl std::error::Error for WalkError {} /// **definition** found (forward declarations are skipped). /// /// `args` are passed verbatim to clang (e.g. `["-std=c++17", "-x", "c++", -/// "-I/path/to/includes"]`). Function bodies are skipped for speed — only -/// declarations are needed for SPO extraction. Parsing tolerates errors -/// (missing includes still yield a partial AST), matching how libclang is -/// used on large real corpora. +/// "-I/path/to/includes"]`). Function bodies ARE parsed, because each method's +/// body arm (the recipe fingerprint: what it writes, reads, throws and +/// dispatches) is harvested alongside its signature; [`walk_tu_configured`] +/// with `None` skips them for a faster signature-only walk. Parsing tolerates +/// errors (missing includes still yield a partial AST), matching how libclang +/// is used on large real corpora. /// /// A **partial** AST is silently possible even when this returns `Ok`: see /// [`walk_tu_with_diagnostics`] for the visibility this function alone does @@ -117,13 +125,38 @@ impl fmt::Display for ParseDiagnostic { pub fn walk_tu_with_diagnostics( path: &Path, args: &[String], +) -> Result<(Vec, Vec), WalkError> { + walk_tu_configured(path, args, Some(&BodyArmConfig::default())) +} + +/// [`walk_tu_with_diagnostics`] with explicit control over the body arm. +/// +/// `arm` decides BOTH what is harvested and how the TU is parsed: +/// +/// - `Some(cfg)` — parse WITH function bodies and fill each [`CppMethod`]'s +/// `writes` / `reads` / `raises` / `calls` / `guarded_writes` from the body, +/// using `cfg`'s mutator vocabulary. This is what [`walk_tu`] and +/// [`walk_tu_with_diagnostics`] do. +/// - `None` — skip function bodies (a faster parse) and leave the arm empty. +/// The signature plane is identical either way, so a consumer that only +/// reconstructs declarations (`ruff_cpp_codegen`) loses nothing by asking +/// for this. +/// +/// # Errors +/// +/// [`WalkError::Libclang`] if libclang fails to initialise; +/// [`WalkError::Parse`] if the TU fails to parse. +pub fn walk_tu_configured( + path: &Path, + args: &[String], + arm: Option<&BodyArmConfig>, ) -> Result<(Vec, Vec), WalkError> { let clang = Clang::new().map_err(WalkError::Libclang)?; let index = Index::new(&clang, false, false); let tu = index .parser(path) .arguments(args) - .skip_function_bodies(true) + .skip_function_bodies(arm.is_none()) .parse() .map_err(|e| WalkError::Parse(e.to_string()))?; @@ -137,7 +170,7 @@ pub fn walk_tu_with_diagnostics( .collect(); let mut out = Vec::new(); - collect_classes(&tu.get_entity(), &mut out); + collect_classes(&tu.get_entity(), &mut out, arm); Ok((out, diagnostics)) } @@ -482,7 +515,7 @@ fn tally_class_bodies(entity: &Entity, hist: &mut BTreeMap) { /// Recurse the AST, emitting a [`CppClass`] for every class/struct /// definition (recursing into namespaces and nested classes). -fn collect_classes(entity: &Entity, out: &mut Vec) { +fn collect_classes(entity: &Entity, out: &mut Vec, arm: Option<&BodyArmConfig>) { for child in entity.get_children() { match child.get_kind() { // Plain classes/structs AND templated classes. libclang FLATTENS a @@ -502,14 +535,14 @@ fn collect_classes(entity: &Entity, out: &mut Vec) { // SPO harvest of a project wants the project's own classes, // never the standard library's internals. if child.is_definition() && !in_system_header(&child) { - if let Some(cls) = build_class(&child) { + if let Some(cls) = build_class(&child, arm) { out.push(cls); } } // Recurse for nested classes regardless of definition state. - collect_classes(&child, out); + collect_classes(&child, out, arm); } - EntityKind::Namespace => collect_classes(&child, out), + EntityKind::Namespace => collect_classes(&child, out, arm), _ => {} } } @@ -518,7 +551,7 @@ fn collect_classes(entity: &Entity, out: &mut Vec) { /// Build a [`CppClass`] from a class/struct definition cursor by reading its /// DIRECT member children (bases, fields, methods). Nested class decls are /// ignored here — [`collect_classes`] emits them separately. -fn build_class(e: &Entity) -> Option { +fn build_class(e: &Entity, arm: Option<&BodyArmConfig>) -> Option { // A `ClassTemplatePartialSpecialization` shares its primary's `get_name()` // (libclang spells it as the bare template name, e.g. `Foo` for // `template class Foo`); using that as-is collides with the @@ -574,7 +607,7 @@ fn build_class(e: &Entity) -> Option { | EntityKind::Destructor | EntityKind::ConversionFunction | EntityKind::FunctionTemplate => { - declarations.push(Declaration::Method(build_method(&m))); + declarations.push(Declaration::Method(build_method(&m, arm))); collect_signature_instantiations(&m, &mut declarations); } // `friend class Foo;` / `friend Ret fn(...);` — the befriended @@ -729,7 +762,12 @@ fn build_base(m: &Entity) -> Option { }) } -fn build_method(m: &Entity) -> CppMethod { +/// Build a [`CppMethod`] from a member-function cursor. `arm` is `Some` when +/// the translation unit was parsed WITH bodies, in which case the body arm is +/// harvested from the method's DEFINITION — which for a method declared in a +/// header and defined in a `.cpp` is a different cursor from `m`, and is the +/// only one that has a body to read. +fn build_method(m: &Entity, arm: Option<&BodyArmConfig>) -> CppMethod { let name = m.get_name().unwrap_or_default(); let is_noexcept = matches!( m.get_exception_specification(), @@ -784,6 +822,9 @@ fn build_method(m: &Entity) -> CppMethod { .flatten() .filter_map(|a| a.get_type().map(|t| t.get_display_name())) .collect(); + let body = arm.map_or_else(BodyArm::default, |cfg| { + method_body_arm(&m.get_definition().unwrap_or(*m), cfg) + }); CppMethod { name, is_pure_virtual: m.is_pure_virtual_method(), @@ -803,164 +844,484 @@ fn build_method(m: &Entity) -> CppMethod { // Public, or unreported (e.g. free function) — default Public. _ => CppAccess::Public, }, + writes: body.writes, + reads: body.reads, + raises: body.raises, + calls: body.calls, + guarded_writes: body.guarded_writes, } } // ───────────────────────────────────────────────────────────────────────── -// DTO ARM (DRAFT — libclang-gated, untested in this checkout: no libclang). +// BODY ARM // // The body-fact fingerprint the fuzzy recipe-codebook needs -// (ruff/.claude/knowledge/fuzzy-recipe-codebook.md §2), for C++ member +// (`ruff/.claude/knowledge/fuzzy-recipe-codebook.md` §2), for C++ member // functions — so the SAME language-agnostic recipe centroids that classify // Rails hooks and C# handlers classify C++ setters / lifecycle overrides. // -// STATUS: reviewed draft. The clang-crate cursor kinds below are correct, but -// this has NOT been run against a real TU (this checkout has no libclang; the -// whole crate is behind the `libclang` feature). A future session with -// LIBCLANG_PATH set should: (1) wire `BodyArm` into `CppMethod` as four -// `Vec` fields + `guarded_writes` (mirroring ruff_spo_triplet::Function), -// (2) emit them in the C++ expand path as writes_field / reads_field / raises / -// calls / writes_if_blank, (3) add a probe on a real corpus (Tesseract) — same -// env-gate + pre-register + drift-fuse discipline as the Ruby/C# legs. +// The arm rides on [`CppMethod`]'s five body fields and expands to the same +// five predicates as a Ruby/Python `Function` body, with the same objects and +// the same truth tiers: `writes_field` / `raises` / `writes_if_blank` are +// Authoritative (the lvalue, the throw type and the guard shape are all +// machine-readable), `reads_field` / `calls` are Inferred (heuristic receiver, +// no scope analysis). +// +// Every cursor shape matched below was measured against libclang 18 on a +// fixture carrying each construct, not inferred from the AST documentation — +// several are counter-intuitive and the tests in `arm_tests` pin them: +// +// `x_ = v` BinaryOperator[ MemberRefExpr(x_), … ] +// `p.x_ = v` BinaryOperator[ MemberRefExpr(x_)[ DeclRefExpr(p) ], … ] +// `this->x_ = v` BinaryOperator[ MemberRefExpr(x_)[ ThisExpr ], … ] +// `x_ += v` CompoundAssignOperator[ MemberRefExpr(x_), … ] +// `++x_` UnaryOperator[ MemberRefExpr(x_) ] +// `arr_[i] = v` BinaryOperator[ ArraySubscriptExpr[ …(arr_), …(i) ], … ] +// `repo_.Save()` CallExpr(Save)[ MemberRefExpr(Save)[ MemberRefExpr(repo_) ] ] +// `name_ = s` CallExpr(operator=)[ MemberRefExpr(name_), …(operator=), …(s) ] +// (a class-typed member assigns through its operator, so it +// is a CallExpr and NOT a BinaryOperator) +// `throw E()` ThrowExpr[ CallExpr(E)[ TypeRef(struct E) ] ] // -// Provenance mapping (matches Function): writes_field / raises / writes_if_blank -// = Authoritative (the lvalue / throw-type / guard shape are machine-readable); -// reads_field / calls = Inferred (heuristic receiver + no scope analysis). -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +// The two shapes that make the difference between a correct fingerprint and a +// plausible-looking wrong one: +// +// * **The operator is not on the cursor.** libclang exposes no +// binary-operator kind, so the draft this replaces treated the LHS of +// ANY binary operator as a write — `if (status_ == v)` recorded a write of +// `status_`. [`binary_operator_spelling`] reads the operator from the +// token stream instead. +// * **An own member is one with no base cursor.** A member of another +// object (`p.x_`) carries its base as a child, and so does a method +// reference (`repo_.Save`), which is why neither is mistaken for this +// class's state. + +/// Which set of calls counts as a lifecycle mutator for the `calls` fact. +/// +/// The C++ analogue of the Ruby `AR_MUTATORS` / C# EF sets, and configurable +/// for the same reason those are: the mutator vocabulary belongs to the +/// framework being harvested, not to the harvester. A `calls` fact fires only +/// for a match, because the signal the triage needs is "does this method +/// dispatch a writer", not a full call graph. +/// +/// [`Self::default`] is the closed set that ships; a corpus with its own +/// persistence vocabulary supplies it through [`Self::with_mutators`] / +/// [`Self::with_mutator_prefixes`]. +#[cfg(feature = "libclang")] +#[derive(Debug, Clone)] +pub struct BodyArmConfig { + mutators: Vec, + mutator_prefixes: Vec, +} + +#[cfg(feature = "libclang")] +impl Default for BodyArmConfig { + fn default() -> Self { + Self { + mutators: [ + "save", "Save", "update", "Update", "insert", "Insert", "remove", "Remove", + "erase", "Erase", "commit", "Commit", "flush", "Flush", "destroy", "Destroy", + "clear", "Clear", "write", "Write", + ] + .iter() + .map(|s| (*s).to_string()) + .collect(), + mutator_prefixes: Vec::new(), + } + } +} + +#[cfg(feature = "libclang")] +impl BodyArmConfig { + /// Replace the exact-match mutator set. + #[must_use] + pub fn with_mutators(mut self, names: impl IntoIterator) -> Self { + self.mutators = names.into_iter().collect(); + self + } + + /// Add prefixes that make any method name starting with one a mutator + /// (e.g. `"Set"` to treat every `SetFoo` as a writer). + #[must_use] + pub fn with_mutator_prefixes(mut self, prefixes: impl IntoIterator) -> Self { + self.mutator_prefixes = prefixes.into_iter().collect(); + self + } + + /// Does a call to `name` count as a lifecycle mutator? + #[must_use] + pub fn is_mutator(&self, name: &str) -> bool { + self.mutators.iter().any(|m| m == name) + || self.mutator_prefixes.iter().any(|p| name.starts_with(p)) + } +} + +/// The recipe fingerprint of one method body, before it is folded into +/// [`CppMethod`]'s own fields. #[cfg(feature = "libclang")] #[derive(Debug, Default, Clone)] pub(crate) struct BodyArm { - pub writes: Vec, // `this->x = …` / `x = …` member assignment - pub reads: Vec, // `this->x` / bare member read - pub raises: Vec, // `throw XError(…)` - pub calls: Vec, // `obj.SaveChanges()` / persistence mutator - pub guarded_writes: Vec, // J1: write under `if (x == nullptr)` etc. + /// `x_ = …` / `x_ += …` / `++x_` — an assignment to an OWN data member. + pub writes: Vec, + /// `x_` read as a value (and the target of a compound assignment). + pub reads: Vec, + /// `throw E(…)` — the BARE type name; the expander adds the `exc:` + /// namespace, exactly as it does for a `Function`. + pub raises: Vec, + /// `repo_.Save()` — a configured lifecycle mutator, as `receiver.method`. + pub calls: Vec, + /// J1: a write under an absence test on that same member. + pub guarded_writes: Vec, } -// The closed persistence-mutator set — the C++ analogue of Ruby's AR_MUTATORS -// and the C# EF set. A `calls` fact fires only for these (the triage needs -// "does it call a writer", not every call). Extend per ORM/framework. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// Which branch of an `if` a condition guards. #[cfg(feature = "libclang")] -fn is_cpp_mutator(name: &str) -> bool { - matches!( - name, - "save" - | "Save" - | "update" - | "Update" - | "insert" - | "Insert" - | "remove" - | "Remove" - | "erase" - | "commit" - | "Commit" - | "flush" - | "Flush" - ) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GuardedBranch { + /// The condition tests for ABSENCE (`x == nullptr`, `!x`, `x.empty()`), + /// so the THEN branch is where the default is written. + Then, + /// The condition tests for PRESENCE (`x != nullptr`, `x`, `!x.empty()`), + /// so the ELSE branch is where the default is written. + Else, } -/// Walk a member-function body cursor and extract the recipe fingerprint. +/// Extract the recipe fingerprint from a member function's body. /// -/// Call with the `Method`/`Constructor`/… entity; it recurses the body via -/// `get_children()`. Local-only J1 guard detection (an `IfStmt` whose condition -/// is a null/empty test on member `X`, containing a write of `X`) — no -/// dominator analysis, keeping `writes_if_blank` Authoritative, exactly as the -/// Ruby `detect_guarded_default` does. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// Call with the entity that HAS the body — [`Entity::get_definition`] where +/// the definition is out of line, the declaration cursor where it is inline. +/// Guard detection is deliberately local (an `IfStmt` whose condition is an +/// absence test on member `X`, threaded into the branch that writes `X`) — no +/// dominator analysis, which is what keeps `writes_if_blank` Authoritative, +/// exactly as the Ruby `detect_guarded_default` does. #[cfg(feature = "libclang")] -pub(crate) fn method_body_arm(method: &Entity) -> BodyArm { +pub(crate) fn method_body_arm(method: &Entity, cfg: &BodyArmConfig) -> BodyArm { let mut arm = BodyArm::default(); - walk_body(method, &mut arm, None); - arm.writes.sort(); - arm.writes.dedup(); - arm.reads.sort(); - arm.reads.dedup(); - arm.raises.sort(); - arm.raises.dedup(); - arm.calls.sort(); - arm.calls.dedup(); - arm.guarded_writes.sort(); - arm.guarded_writes.dedup(); + walk_body(method, &mut arm, cfg, None); + for facts in [ + &mut arm.writes, + &mut arm.reads, + &mut arm.raises, + &mut arm.calls, + &mut arm.guarded_writes, + ] { + facts.sort(); + facts.dedup(); + } arm } -// `guard` = the member name the enclosing branch is null/empty-guarded on (J1), -// threaded down only into that branch. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// Walk every CHILD of `node`. `guard` is the member the enclosing branch is +/// absence-guarded on (J1), threaded down only into that branch. #[cfg(feature = "libclang")] -fn walk_body(node: &Entity, arm: &mut BodyArm, guard: Option<&str>) { +fn walk_body(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { for child in node.get_children() { - match child.get_kind() { - // `throw XError("…")` — the exception type name is the throw's - // sub-expression type. `CXXThrowExpr` wraps the constructed value. - EntityKind::ThrowExpr => { - if let Some(ty) = thrown_type_name(&child) { - arm.raises.push(format!("exc:{ty}")); - } + walk_node(&child, arm, cfg, guard); + } +} + +/// Walk ONE node, matching it before descending. +#[cfg(feature = "libclang")] +fn walk_node(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { + match node.get_kind() { + EntityKind::ThrowExpr => { + if let Some(ty) = thrown_type_name(node) { + arm.raises.push(ty); } - // `a = b` — a BinaryOperator whose operator is `=`. The clang crate - // does not expose the operator token directly on stable, so the - // idiom is: the first child is the lvalue. If it is a member ref - // (`this->x` / `x`), it is a write of that member; a J1 guard makes - // it a guarded (default) write. - EntityKind::BinaryOperator => { - if let Some(member) = child - .get_children() - .first() - .filter(|c| c.get_kind() == EntityKind::MemberRefExpr) - .and_then(Entity::get_name) - { - arm.writes.push(member.clone()); - if guard == Some(member.as_str()) { - arm.guarded_writes.push(member); - } + walk_body(node, arm, cfg, guard); + } + // Only `=` is an assignment. Every other binary operator (`==`, `+`, + // `<<`, …) reads both sides — which is why the operator has to be read + // off the tokens rather than assumed from the cursor kind. + EntityKind::BinaryOperator => { + let children = node.get_children(); + if let Some((lhs, rest)) = children.split_first() + && binary_operator_spelling(node).as_deref() == Some("=") + { + record_write(arm, lhs, guard); + // The LHS's own member reference is the write TARGET, not a + // read, so descend past it — but keep whatever it contains + // (`arr_[i]`'s subscript, `p` in `p.x_`). + walk_body(lhs, arm, cfg, guard); + for r in rest { + walk_node(r, arm, cfg, guard); } - // Recurse into the RHS for nested reads/calls/raises. - walk_body(&child, arm, guard); + } else { + walk_body(node, arm, cfg, guard); } - // `obj.method(...)` — a persistence-mutator dispatch → `calls`. - EntityKind::CallExpr => { - if let Some(name) = child.get_name() - && is_cpp_mutator(&name) - { - // "receiver.method": the receiver display name if resolvable, - // else `self`. (Heuristic — Inferred tier, like Ruby.) - let recv = call_receiver(&child).unwrap_or_else(|| "self".to_string()); - arm.calls.push(format!("{recv}.{name}")); + } + // `x_ += v` is both a write and a read of `x_` — a read-modify-write. + EntityKind::CompoundAssignOperator => { + let children = node.get_children(); + if let Some((lhs, rest)) = children.split_first() { + record_write(arm, lhs, guard); + if let Some(m) = assignment_target(lhs) { + arm.reads.push(m); + } + walk_body(lhs, arm, cfg, guard); + for r in rest { + walk_node(r, arm, cfg, guard); } - walk_body(&child, arm, guard); } - // `this->x` / bare `x` as a value → a member read. (The lvalue of an - // assignment is handled above and NOT double-counted here, because - // this arm only fires for a MemberRefExpr that is not the direct - // first child of a BinaryOperator — see the C# LHS-exclusion note.) - EntityKind::MemberRefExpr => { - if let Some(name) = child.get_name() { - arm.reads.push(name); + } + // `++x_` / `x_--` are read-modify-writes too; every other unary + // operator (`!x_`, `*x_`, `-x_`) only reads. + EntityKind::UnaryOperator => { + let is_inc_dec = unary_operator_is_inc_dec(node); + let children = node.get_children(); + if let Some(operand) = children.first() + && is_inc_dec + { + record_write(arm, operand, guard); + if let Some(m) = assignment_target(operand) { + arm.reads.push(m); } + walk_body(operand, arm, cfg, guard); + } else { + walk_body(node, arm, cfg, guard); + } + } + EntityKind::CallExpr => call_expr(node, arm, cfg, guard), + EntityKind::IfStmt => if_stmt(node, arm, cfg, guard), + // A member reference that is not an assignment target is a read — but + // only when it names an OWN data member. `p.x_` and `repo_.Save` both + // carry their base as a child and are therefore not own members; + // descending still finds `repo_` inside the latter. + EntityKind::MemberRefExpr => { + if let Some(name) = own_member_name(node) { + arm.reads.push(name); + } + walk_body(node, arm, cfg, guard); + } + _ => walk_body(node, arm, cfg, guard), + } +} + +/// A call: either an overloaded-operator assignment (a write), a configured +/// lifecycle mutator (a `calls` fact), or an ordinary call walked for its +/// arguments. +#[cfg(feature = "libclang")] +fn call_expr(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { + let name = node.get_name().unwrap_or_default(); + let children = node.get_children(); + + // A class-typed member assigns through `operator=`, so `name_ = s` arrives + // as a CallExpr whose FIRST child is the assigned object (not a callee + // reference, which is the shape an ordinary method call has). + if let Some(op) = name.strip_prefix("operator") + && op.ends_with('=') + && !matches!(op, "==" | "!=" | "<=" | ">=") + && let Some((lhs, rest)) = children.split_first() + { + record_write(arm, lhs, guard); + // A compound form (`+=`, `|=`, …) also reads its target. + if op != "=" + && let Some(m) = assignment_target(lhs) + { + arm.reads.push(m); + } + walk_body(lhs, arm, cfg, guard); + for r in rest { + walk_node(r, arm, cfg, guard); + } + return; + } + + if cfg.is_mutator(&name) { + arm.calls.push(format!( + "{}.{name}", + call_receiver(node).as_deref().unwrap_or("this") + )); + } + for child in &children { + // The callee reference names the METHOD, not a data member — walking + // it as an ordinary node would record `Save` as a read of a field that + // does not exist. Its children (the receiver) still matter. + if child.get_kind() == EntityKind::MemberRefExpr && child.get_name().as_ref() == Some(&name) + { + walk_body(child, arm, cfg, guard); + } else { + walk_node(child, arm, cfg, guard); + } + } +} + +/// An `if`: walk the condition for its reads, then each branch — threading the +/// J1 guard into whichever branch the condition proves the member ABSENT in. +#[cfg(feature = "libclang")] +fn if_stmt(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { + let children = node.get_children(); + let Some((cond, branches)) = children.split_first() else { + return; + }; + walk_node(cond, arm, cfg, None); + // `branches` is [then] or [then, else]. + let detected = absence_guard(cond); + for (i, branch) in branches.iter().enumerate() { + let branch_guard = match &detected { + // This `if` decides the guard for both of its branches: the one + // the condition proves the member absent in gets it, the other + // gets nothing (an enclosing guard is dropped rather than + // reasoned about — the safe direction, since a missed guard only + // records a plain write). + Some((member, GuardedBranch::Then)) => (i == 0).then_some(member.as_str()), + Some((member, GuardedBranch::Else)) => (i == 1).then_some(member.as_str()), + // No guard here — an enclosing one still holds in both branches. + None => guard, + }; + walk_node(branch, arm, cfg, branch_guard); + } +} + +/// Record an assignment to `lhs` as a write, and as a J1 guarded write when +/// the enclosing branch is absence-guarded on that same member. +#[cfg(feature = "libclang")] +fn record_write(arm: &mut BodyArm, lhs: &Entity, guard: Option<&str>) { + if let Some(member) = assignment_target(lhs) { + if guard == Some(member.as_str()) { + arm.guarded_writes.push(member.clone()); + } + arm.writes.push(member); + } +} + +/// The own data member an assignment's left-hand side ultimately names, seeing +/// through the wrappers libclang inserts plus subscripting and dereference +/// (`arr_[i] = v` and `*ptr_ = v` both write the member). `None` when the +/// target is anything else — a local, a parameter, or another object's member. +#[cfg(feature = "libclang")] +fn assignment_target(lhs: &Entity) -> Option { + let mut cur = *lhs; + // Bounded: each step descends one AST level, and the depth of an lvalue + // expression is finite. The cap only guards against a cyclic cursor graph. + for _ in 0..32 { + match cur.get_kind() { + EntityKind::MemberRefExpr => return own_member_name(&cur), + EntityKind::UnexposedExpr + | EntityKind::ParenExpr + | EntityKind::ArraySubscriptExpr + | EntityKind::UnaryOperator => { + cur = cur.get_children().into_iter().next()?; } - // Structural wrappers (incl. `IfStmt`, `CompoundStmt`, - // `UnexposedExpr`) — recurse so facts inside them are matched as - // children. NOTE: unlike Ruby/C#, C++ J1 (`writes_if_blank`) is a - // documented FOLLOW-UP here: the libclang AST wraps the guard cond - // and the guarded write in `UnexposedExpr` nodes (see the cursor - // dump in `examples/`), so robust `if (x == nullptr) x = v` guard - // detection needs an UnexposedExpr-aware pass. Until then C++ - // `guarded_writes` stays empty (a write-if-blank is recorded as a - // plain write → classified Compute/Normalize, never a false - // essential — the safe direction). `null_guarded_member` is the - // seed for that follow-up. - _ => walk_body(&child, arm, guard), + _ => return None, } } + None } -// The thrown exception's type name. `throw X(...)` nests the operand under -// UnexposedExpr/ConstructExpr wrappers, so recurse for the first node yielding -// a concrete, non-void type name. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// The member name when `e` references a data member of THIS object. +/// +/// An implicit `this` base is not a visited child, and an explicit one is a +/// [`EntityKind::ThisExpr`]; any other base (`p.x_`, `repo_.Save`) means the +/// reference belongs to something else. +#[cfg(feature = "libclang")] +fn own_member_name(e: &Entity) -> Option { + if e.get_kind() != EntityKind::MemberRefExpr { + return None; + } + match e.get_children().first().map(Entity::get_kind) { + None | Some(EntityKind::ThisExpr) => e.get_name(), + _ => None, + } +} + +/// The operator spelling of a binary operator, read off the token stream. +/// +/// libclang exposes no binary-operator kind, so the operator is the first +/// punctuation token that starts at or after the end of the left operand. +#[cfg(feature = "libclang")] +fn binary_operator_spelling(node: &Entity) -> Option { + let lhs_end = node + .get_children() + .first()? + .get_range()? + .get_end() + .get_file_location() + .offset; + node.get_range()? + .tokenize() + .into_iter() + .find(|t| { + t.get_kind() == clang::token::TokenKind::Punctuation + && t.get_location().get_file_location().offset >= lhs_end + }) + .map(|t| t.get_spelling()) +} + +/// Is this unary operator an increment or decrement (prefix or postfix)? +#[cfg(feature = "libclang")] +fn unary_operator_is_inc_dec(node: &Entity) -> bool { + let Some(range) = node.get_range() else { + return false; + }; + let tokens = range.tokenize(); + let is_inc_dec = |t: Option<&clang::token::Token<'_>>| { + t.map(clang::token::Token::get_spelling) + .is_some_and(|s| s == "++" || s == "--") + }; + is_inc_dec(tokens.first()) || is_inc_dec(tokens.last()) +} + +/// The J1 fact's condition half: does this `if` condition test ONE own member +/// for absence or presence, and which branch does that make the guarded one? +/// +/// Deliberately conservative — a compound condition (`&&` / `||`), a condition +/// naming more than one own member, or a shape not in the table below yields +/// `None`, so the write is recorded as a plain write. That is the safe +/// direction: a missed guard classifies the method as `Compute`/`Normalize`, +/// never as a false schema default. +#[cfg(feature = "libclang")] +fn absence_guard(cond: &Entity) -> Option<(String, GuardedBranch)> { + let mut members = Vec::new(); + collect_own_members(cond, &mut members); + members.sort(); + members.dedup(); + let [member] = members.as_slice() else { + return None; + }; + + let tokens: Vec = cond + .get_range()? + .tokenize() + .into_iter() + .map(|t| t.get_spelling()) + .collect(); + let has = |t: &str| tokens.iter().any(|s| s == t); + if has("&&") || has("||") { + return None; + } + let null_literal = has("nullptr") || has("NULL") || has("0") || has("false"); + let empty_test = has("empty") || has("isEmpty") || has("IsEmpty"); + let leading_bang = tokens.first().is_some_and(|t| t == "!"); + let comparison = ["==", "!=", "<", ">", "<=", ">="].iter().any(|t| has(t)); + + let branch = match () { + () if has("==") && null_literal => GuardedBranch::Then, + () if has("!=") && null_literal => GuardedBranch::Else, + () if leading_bang && empty_test => GuardedBranch::Else, + () if leading_bang => GuardedBranch::Then, + () if empty_test => GuardedBranch::Then, + // A bare `if (x_)` is a presence test. + () if !comparison && !empty_test => GuardedBranch::Else, + () => return None, + }; + Some((member.clone(), branch)) +} + +/// Every own data member referenced anywhere under `node`. +#[cfg(feature = "libclang")] +fn collect_own_members(node: &Entity, out: &mut Vec) { + if let Some(name) = own_member_name(node) { + out.push(name); + } + for child in node.get_children() { + collect_own_members(&child, out); + } +} + +/// The thrown exception's type name. `throw X(...)` nests the operand under +/// wrapper cursors, so recurse for the first node yielding a concrete, +/// non-void type name. #[cfg(feature = "libclang")] fn thrown_type_name(throw: &Entity) -> Option { fn first_typed(e: &Entity) -> Option { @@ -975,41 +1336,28 @@ fn thrown_type_name(throw: &Entity) -> Option { throw.get_children().iter().find_map(first_typed) } -// `x == nullptr` / `x == 0` / `x.empty()` → the guarded member `X`. Retained as -// the SEED for the C++ J1 follow-up (see the IfStmt note in `walk_body`); not -// yet wired, hence `dead_code`. -#[allow(dead_code)] -// Draft: the -// clang crate surfaces the operator via the child token stream; a full impl -// inspects `get_children()` for a MemberRefExpr paired with a null literal. -#[cfg(feature = "libclang")] -fn null_guarded_member(cond: &Entity) -> Option { - // Look for a MemberRefExpr anywhere in the condition whose sibling is a - // null/zero literal or whose parent call is `.empty()`. Kept deliberately - // conservative (only the clear cases) so the fact stays Authoritative. - cond.get_children() - .iter() - .find(|c| c.get_kind() == EntityKind::MemberRefExpr) - .and_then(Entity::get_name) -} - -// Best-effort receiver label for a call (`obj` in `obj.save()`), else None. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// The receiver of a method call (`repo_` in `repo_.Save()`), or `None` for an +/// implicit-`this` call — the callee reference's own base. #[cfg(feature = "libclang")] fn call_receiver(call: &Entity) -> Option { - call.get_children() - .iter() - .find(|c| { - matches!( - c.get_kind(), - EntityKind::MemberRefExpr | EntityKind::DeclRefExpr - ) - }) - .and_then(Entity::get_name) + let name = call.get_name()?; + let callee = call.get_children().into_iter().find(|c| { + c.get_kind() == EntityKind::MemberRefExpr && c.get_name() == Some(name.clone()) + })?; + let mut cur = callee.get_children().into_iter().next()?; + for _ in 0..32 { + match cur.get_kind() { + EntityKind::MemberRefExpr | EntityKind::DeclRefExpr => return cur.get_name(), + EntityKind::UnexposedExpr | EntityKind::ParenExpr => { + cur = cur.get_children().into_iter().next()?; + } + _ => return None, + } + } + None } -// `List` / `foo::Bar` → a stable bare type name for the `exc:` object. -#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand is the follow-up +/// `List` / `foo::Bar` → a stable bare type name for the `raises` object. #[cfg(feature = "libclang")] fn bare_type_name(display: &str) -> String { let s = display @@ -1034,94 +1382,374 @@ mod arm_tests { use super::*; use std::io::Write; - fn find_method<'a>(e: &Entity<'a>, name: &str) -> Option> { + /// Every method under `e`, by name. + fn methods<'a>(e: &Entity<'a>, out: &mut BTreeMap>) { for c in e.get_children() { - if c.get_kind() == EntityKind::Method && c.get_name().as_deref() == Some(name) { - return Some(c); - } - if let Some(f) = find_method(&c, name) { - return Some(f); + if matches!( + c.get_kind(), + EntityKind::Method | EntityKind::Constructor | EntityKind::Destructor + ) && let Some(name) = c.get_name() + { + // A method seen twice (header declaration + out-of-line + // definition) keeps the DEFINITION, which is the cursor the + // arm needs — `build_method` resolves the same way. + if c.is_definition() || !out.contains_key(&name) { + out.insert(name, c); + } } + methods(&c, out); } - None } - // Parse an inline C++ fixture WITH function bodies (walk_tu skips them), - // find one method by name, and return its recipe fingerprint. - fn arm_of(src: &str, method: &str) -> BodyArm { - let dir = std::env::temp_dir().join(format!("cpp_arm_{method}")); - std::fs::create_dir_all(&dir).unwrap(); + /// Parse one inline C++ fixture WITH bodies and return every method's arm. + /// + /// One parse per fixture, not one per method: `Clang` is a process + /// singleton, so each parse serialises on [`CLANG_TEST_LOCK`]. + fn arms_with(name: &str, src: &str, cfg: &BodyArmConfig) -> BTreeMap { + let dir = std::env::temp_dir().join(format!("cpp_arm_{name}")); + std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); - let mut fh = std::fs::File::create(&path).unwrap(); - fh.write_all(src.as_bytes()).unwrap(); + let mut fh = std::fs::File::create(&path).expect("fixture file"); + fh.write_all(src.as_bytes()).expect("write fixture"); drop(fh); + let _guard = CLANG_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let clang = Clang::new().unwrap(); + let clang = Clang::new().expect("libclang"); let index = Index::new(&clang, false, false); let tu = index .parser(&path) .arguments(&["-std=c++17".to_string()]) - .skip_function_bodies(false) // ← the arm needs bodies + .skip_function_bodies(false) .parse() - .unwrap(); - let m = find_method(&tu.get_entity(), method).expect("method not found"); - method_body_arm(&m) + .expect("parse"); + let mut found = BTreeMap::new(); + methods(&tu.get_entity(), &mut found); + found + .into_iter() + .map(|(n, e)| (n, method_body_arm(&e, cfg))) + .collect() } - #[test] - fn cpp_body_arm_extracts_the_fingerprint() { - // Declaration order matters: `BadStatus` and `Repo` must be complete - // types before `Patient` uses them (a forward-ref leaves the throw - // operand unresolved and the arm sees no type). - let src = r#" + fn arms(name: &str, src: &str) -> BTreeMap { + arms_with(name, src, &BodyArmConfig::default()) + } + + /// A class exercising every write/read shape the arm must tell apart. + /// Declaration order matters: a type must be complete before a member + /// uses it, or the operand's type is unresolved and the arm sees nothing. + const SHAPES: &str = r#" struct BadStatus {}; -struct Repo { void save(); }; +struct Repo { void Save(); void Peek() const; }; +struct Str { bool empty() const; }; struct Patient { int status_; + int arr_[4]; + int* ptr_; Repo repo_; - // normalize: unconditional self-write - void tidy() { status_ = status_ + 1; } - // guard: throw only - void validate() { if (status_ == 0) throw BadStatus(); } - // cascade: mutator dispatch - void persist() { repo_.save(); } + Str name_; + void set(int v) { status_ = v; } + void compare(int v) { if (status_ == v) { } } + void compound(int v) { status_ += v; } + void increment() { ++status_; } + void decrement() { status_--; } + void assign_object(Str s) { name_ = s; } + void other_object(Patient& p, int v) { p.status_ = v; } + void explicit_this(int v) { this->status_ = v; } + void subscript(int i, int v) { arr_[i] = v; } + void read_only(int& out) { out = status_; } + void persist() { repo_.Save(); } + void peek() { repo_.Peek(); } + void call_self() { reset(); } + void reset() { status_ = 0; } + void thrower() { throw BadStatus(); } }; "#; - let tidy = arm_of(src, "tidy"); - assert!( - tidy.writes.contains(&"status_".to_string()), - "writes {:?}", - tidy.writes - ); - assert!( - tidy.reads.contains(&"status_".to_string()), - "reads {:?}", - tidy.reads - ); - let validate = arm_of(src, "validate"); - assert!( - validate.writes.is_empty(), - "guard writes nothing: {:?}", - validate.writes - ); + #[test] + fn a_plain_setter_writes_its_member_without_reading_it() { + let a = &arms("shapes", SHAPES)["set"]; + assert_eq!(a.writes, ["status_"]); assert!( - validate.raises.iter().any(|r| r.contains("BadStatus")), - "raises {:?}", - validate.raises + a.reads.is_empty(), + "the assignment target is not a read: {:?}", + a.reads ); + } - let persist = arm_of(src, "persist"); - assert!( - persist - .calls + /// The defect the token-based operator lookup exists to fix: without it + /// every binary operator's left operand read as a write, so a pure + /// comparison reported a mutation. + #[test] + fn a_comparison_is_a_read_and_never_a_write() { + let a = &arms("shapes", SHAPES)["compare"]; + assert!(a.writes.is_empty(), "comparison wrote {:?}", a.writes); + assert_eq!(a.reads, ["status_"]); + } + + #[test] + fn a_read_modify_write_is_both() { + let all = arms("shapes", SHAPES); + for name in ["compound", "increment", "decrement"] { + let a = &all[name]; + assert_eq!(a.writes, ["status_"], "{name} writes"); + assert_eq!(a.reads, ["status_"], "{name} reads"); + } + } + + /// A class-typed member assigns through `operator=`, so libclang reports + /// the assignment as a call — a shape a BinaryOperator-only walker misses + /// entirely. + #[test] + fn an_overloaded_assignment_is_still_a_write() { + let a = &arms("shapes", SHAPES)["assign_object"]; + assert_eq!(a.writes, ["name_"]); + assert!(a.reads.is_empty(), "copy-assign read {:?}", a.reads); + } + + #[test] + fn another_objects_member_is_not_state_of_this_class() { + let a = &arms("shapes", SHAPES)["other_object"]; + assert!(a.writes.is_empty(), "wrote {:?}", a.writes); + assert!(a.reads.is_empty(), "read {:?}", a.reads); + } + + #[test] + fn an_explicit_this_is_the_same_member_as_an_implicit_one() { + let all = arms("shapes", SHAPES); + assert_eq!(all["explicit_this"].writes, all["set"].writes); + assert_eq!(all["explicit_this"].reads, all["set"].reads); + } + + #[test] + fn a_subscripted_assignment_writes_the_array_member() { + let a = &arms("shapes", SHAPES)["subscript"]; + assert_eq!(a.writes, ["arr_"]); + } + + #[test] + fn a_value_use_is_a_read() { + let a = &arms("shapes", SHAPES)["read_only"]; + assert_eq!(a.reads, ["status_"]); + assert!(a.writes.is_empty(), "wrote {:?}", a.writes); + } + + /// The receiver is the object the mutator is called ON — not the method + /// name, which is what the callee cursor carries. + #[test] + fn a_mutator_call_names_its_receiver() { + let a = &arms("shapes", SHAPES)["persist"]; + assert_eq!(a.calls, ["repo_.Save"]); + assert_eq!(a.reads, ["repo_"], "the receiver is read"); + } + + #[test] + fn a_non_mutator_call_is_not_a_calls_fact() { + let a = &arms("shapes", SHAPES)["peek"]; + assert!(a.calls.is_empty(), "calls {:?}", a.calls); + } + + /// A method reference is not a data member, so calling one on this object + /// must not manufacture a read of a field that does not exist. + #[test] + fn calling_own_method_does_not_read_a_field_named_after_it() { + let a = &arms("shapes", SHAPES)["call_self"]; + assert!(a.reads.is_empty(), "read {:?}", a.reads); + assert!(a.writes.is_empty(), "wrote {:?}", a.writes); + } + + /// The IR carries the BARE type name; the `exc:` namespace is the + /// expander's job, exactly as it is for a `Function`. + #[test] + fn a_throw_records_the_bare_exception_type() { + let a = &arms("shapes", SHAPES)["thrower"]; + assert_eq!(a.raises, ["BadStatus"]); + } + + const GUARDS: &str = r#" +struct Str { bool empty() const; }; +struct Cfg { + int* ptr_; + Str name_; + int other_; + void null_guard(int* v) { if (ptr_ == nullptr) { ptr_ = v; } } + void bang_guard(int* v) { if (!ptr_) { ptr_ = v; } } + void empty_guard(Str s) { if (name_.empty()) { name_ = s; } } + void present_guard(int* v) { if (ptr_ != nullptr) { } else { ptr_ = v; } } + void bare_truth_guard(int* v) { if (ptr_) { } else { ptr_ = v; } } + void not_empty_guard(Str s) { if (!name_.empty()) { } else { name_ = s; } } + void unguarded(int* v) { ptr_ = v; } + void wrong_branch(int* v) { if (ptr_ == nullptr) { } else { ptr_ = v; } } + void other_member_guard(int* v) { if (other_ == 0) { ptr_ = v; } } + void compound_condition(int* v, bool b) { if (ptr_ == nullptr && b) { ptr_ = v; } } +}; +"#; + + #[test] + fn an_absence_test_makes_the_then_branch_write_a_guarded_write() { + let all = arms("guards", GUARDS); + for name in ["null_guard", "bang_guard", "empty_guard"] { + let a = &all[name]; + assert_eq!(a.guarded_writes.len(), 1, "{name}: {:?}", a.guarded_writes); + assert!( + a.writes.contains(&a.guarded_writes[0]), + "{name}: a guarded write is always a write too" + ); + } + } + + /// The mirror image: when the condition proves the member PRESENT, the + /// default is written in the `else`, so that is the guarded branch. + #[test] + fn a_presence_test_makes_the_else_branch_write_a_guarded_write() { + let all = arms("guards", GUARDS); + for (name, member) in [ + ("present_guard", "ptr_"), + ("bare_truth_guard", "ptr_"), + ("not_empty_guard", "name_"), + ] { + let a = &all[name]; + assert_eq!(a.guarded_writes, [member], "{name} guarded writes"); + assert_eq!(a.writes, [member], "{name} writes"); + } + } + + /// The silence half. Each of these writes the member and must NOT be + /// recorded as a schema default — a guard that fires on everything + /// carries exactly as much information as one that never fires. + #[test] + fn a_write_that_is_not_absence_guarded_stays_a_plain_write() { + let all = arms("guards", GUARDS); + for name in [ + // No conditional at all. + "unguarded", + // Guarded, but the write is in the branch where the member is + // known PRESENT. + "wrong_branch", + // The condition tests a DIFFERENT member. + "other_member_guard", + // A compound condition is not analysed. + "compound_condition", + ] { + let a = &all[name]; + assert!(!a.writes.is_empty(), "{name} should still write"); + assert!( + a.guarded_writes.is_empty(), + "{name} claimed a guarded write: {:?}", + a.guarded_writes + ); + } + } + + #[test] + fn the_mutator_vocabulary_is_configurable() { + let src = r#" +struct Repo { void Persist(); void Save(); }; +struct Svc { Repo repo_; void go() { repo_.Persist(); repo_.Save(); } }; +"#; + // The shipped set does not know `Persist`. + let default = &arms("mutator_default", src)["go"]; + assert_eq!(default.calls, ["repo_.Save"]); + + // …and a corpus with its own vocabulary can say so. + let custom = BodyArmConfig::default().with_mutators(["Persist".to_string()]); + let configured = &arms_with("mutator_custom", src, &custom)["go"]; + assert_eq!(configured.calls, ["repo_.Persist"]); + + // Prefixes work too, and compose with the exact set. + let prefixed = BodyArmConfig::default().with_mutator_prefixes(["Per".to_string()]); + let both = &arms_with("mutator_prefix", src, &prefixed)["go"]; + assert_eq!(both.calls, ["repo_.Persist", "repo_.Save"]); + } + + /// The reason `build_method` resolves through [`Entity::get_definition`]: + /// a method declared in a class and defined out of line has NO body on the + /// declaration cursor, which is the one `build_class` walks. + #[test] + fn an_out_of_line_definition_is_harvested_through_the_declaration() { + let src = r#" +struct Repo { void Save(); }; +struct Svc { + int status_; + Repo repo_; + void finish(int v); +}; +void Svc::finish(int v) { status_ = v; repo_.Save(); } +"#; + let dir = std::env::temp_dir().join("cpp_arm_outofline"); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("f.cpp"); + std::fs::write(&path, src).expect("write fixture"); + + let _guard = CLANG_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (classes, _) = walk_tu_configured( + &path, + &["-std=c++17".to_string()], + Some(&BodyArmConfig::default()), + ) + .expect("walk"); + + let svc = classes.iter().find(|c| c.name == "Svc").expect("Svc"); + let finish = svc + .declarations + .iter() + .find_map(|d| match d { + Declaration::Method(m) if m.name == "finish" => Some(m), + _ => None, + }) + .expect("finish"); + assert_eq!(finish.writes, ["status_"]); + assert_eq!(finish.calls, ["repo_.Save"]); + } + + /// The opt-out: a signature-only walk parses no bodies, so every arm is + /// empty while the signature plane is unchanged. + /// + /// The opt-out is enforced TWICE — the TU is parsed with + /// `skip_function_bodies`, and `build_method` does not harvest — so + /// disabling either mechanism alone leaves this test green. That is + /// redundancy, not a vacuous assertion: it goes red when the parse flag is + /// forced on, and red again when both are forced on. Anyone tempted to + /// drop one of the two as dead weight should expect no test to notice. + #[test] + fn a_signature_only_walk_leaves_the_arm_empty() { + let src = r#" +struct Svc { int status_; void set(int v) { status_ = v; } }; +"#; + let dir = std::env::temp_dir().join("cpp_arm_sigonly"); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("f.cpp"); + std::fs::write(&path, src).expect("write fixture"); + let args = ["-std=c++17".to_string()]; + + let _guard = CLANG_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let method_of = |arm: Option<&BodyArmConfig>| { + let (classes, _) = walk_tu_configured(&path, &args, arm).expect("walk"); + classes .iter() - .any(|c| c.split('.').next_back() == Some("save")), - "calls {:?}", - persist.calls - ); + .find(|c| c.name == "Svc") + .and_then(|c| { + c.declarations.iter().find_map(|d| match d { + Declaration::Method(m) if m.name == "set" => Some(m.clone()), + _ => None, + }) + }) + .expect("set") + }; + let with_arm = method_of(Some(&BodyArmConfig::default())); + let without = method_of(None); + + assert_eq!(with_arm.writes, ["status_"], "the arm is the default"); + assert!(without.writes.is_empty(), "opting out harvests no body"); + // The signature plane is identical either way. + assert_eq!(with_arm.param_types, without.param_types); + assert_eq!(with_arm.is_const, without.is_const); + assert_eq!(with_arm.access, without.access); } } diff --git a/crates/ruff_cpp_spo/src/lib.rs b/crates/ruff_cpp_spo/src/lib.rs index 6f7fc68d9b0b8e..8ed377ce0d332e 100644 --- a/crates/ruff_cpp_spo/src/lib.rs +++ b/crates/ruff_cpp_spo/src/lib.rs @@ -59,8 +59,8 @@ use ruff_spo_triplet::{ mod clang_walker; #[cfg(feature = "libclang")] pub use clang_walker::{ - MAPPED_CURSOR_KINDS, ParseDiagnostic, WalkError, class_body_cursor_histogram, walk_enums, - walk_free_functions, walk_tu, walk_tu_with_diagnostics, + BodyArmConfig, MAPPED_CURSOR_KINDS, ParseDiagnostic, WalkError, class_body_cursor_histogram, + walk_enums, walk_free_functions, walk_tu, walk_tu_configured, walk_tu_with_diagnostics, }; /// The namespace prefix for C++ machine-plane subjects/objects. @@ -311,8 +311,14 @@ fn walk_files(files: &[std::path::PathBuf], args: &[String]) -> Result { for cls in classes { - seen.entry(cls.qualified_name()) - .or_insert_with(|| model_from_class(&cls)); + match seen.entry(cls.qualified_name()) { + std::collections::btree_map::Entry::Vacant(e) => { + e.insert(model_from_class(&cls)); + } + std::collections::btree_map::Entry::Occupied(mut e) => { + merge_body_arms(e.get_mut(), &model_from_class(&cls)); + } + } } } Err(WalkError::Parse(_)) => {} @@ -324,6 +330,46 @@ fn walk_files(files: &[std::path::PathBuf], args: &[String]) -> Result Some(m), + _ => None, + }) + }) + .expect("the header declares finish"); + assert!( + header_finish.writes.is_empty(), + "the header sighting must be arm-less: {:?}", + header_finish.writes + ); + + let graph = extract_dir(&dir, &args).expect("libclang init (LIBCLANG_PATH set)"); + let svc = graph + .models + .iter() + .find(|m| m.name == "Svc") + .expect("Svc harvested"); + let finish = svc + .methods + .iter() + .find(|m| m.name == "finish") + .expect("finish harvested"); + assert_eq!(finish.writes, ["status_"], "the out-of-line body's write"); + assert_eq!(finish.calls, ["repo_.Save"], "the out-of-line body's call"); + // The signature plane is unaffected by the merge. + assert_eq!(finish.param_types, ["int"]); + assert_eq!(svc.methods.len(), 1, "the class is not duplicated"); + } + /// First **ndjson emission** from a real corpus subset — gated on /// `TESSERACT_SRC`. Walks all of `src/ccutil` via [`extract_dir`], expands /// to SPO triples, serialises to ndjson, and round-trips it. The diff --git a/crates/ruff_spo_triplet/src/expand.rs b/crates/ruff_spo_triplet/src/expand.rs index 5368bdda75dea2..1620430dc2c63c 100644 --- a/crates/ruff_spo_triplet/src/expand.rs +++ b/crates/ruff_spo_triplet/src/expand.rs @@ -888,6 +888,52 @@ impl Expander { .to_string(), Provenance::CppExtracted, ); + // Body arm — the `Function` quartet + J1 fact, SAME predicates, SAME + // object encoding, SAME truth tiers as the Ruby/Python path above, so + // `recipe::classify` and OGAR `lift_actions` see one shape. Own-member + // objects join the `cpp_field` node (`{model_iri}.{member}`); `raises` + // takes the `exc:` namespace here (the IR carries the bare type name); + // `calls` stays a verbatim `"receiver.method"` string, not an IRI. + for read in &method.reads { + self.push( + method_iri.clone(), + Predicate::ReadsField, + format!("{model_iri}.{read}"), + Provenance::Inferred, + ); + } + for exc in &method.raises { + self.push( + method_iri.clone(), + Predicate::Raises, + format!("exc:{exc}"), + Provenance::Authoritative, + ); + } + for write in &method.writes { + self.push( + method_iri.clone(), + Predicate::WritesField, + format!("{model_iri}.{write}"), + Provenance::Authoritative, + ); + } + for gw in &method.guarded_writes { + self.push( + method_iri.clone(), + Predicate::WritesIfBlank, + format!("{model_iri}.{gw}"), + Provenance::Authoritative, + ); + } + for call in &method.calls { + self.push( + method_iri.clone(), + Predicate::Calls, + call.clone(), + Provenance::Inferred, + ); + } if let Some(req) = &method.requires_clause { // Last potential use of `method_iri` — move, don't clone. self.push( @@ -2391,6 +2437,11 @@ mod tests { is_const: true, is_static: false, access: CppAccess::Public, + writes: vec!["status_".to_string(), "recognizer_".to_string()], + reads: vec!["recognizer_".to_string()], + raises: vec!["BadStatus".to_string()], + calls: vec!["repo_.Save".to_string()], + guarded_writes: vec!["recognizer_".to_string()], }); rec.methods.push(CppMethod { name: "Clear".to_string(), @@ -2405,6 +2456,11 @@ mod tests { is_const: false, is_static: false, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.methods.push(CppMethod { name: "kMaxRating".to_string(), @@ -2419,6 +2475,11 @@ mod tests { is_const: false, is_static: true, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.methods.push(CppMethod { name: "operator==".to_string(), @@ -2433,6 +2494,11 @@ mod tests { is_const: false, is_static: false, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.templates.push(CppTemplate { kind: CppTemplateKind::Specialisation, @@ -2548,6 +2614,87 @@ mod tests { // Inferred per-edge overrides assert_eq!(truth("uses_macro_expansion"), Some((0.85, 0.75))); assert_eq!(truth("template_instantiates"), Some((0.85, 0.75))); + // Body arm — the C++ method reuses the `Function` tiers verbatim, so + // the recipe classifier sees one calibration regardless of frontend. + // The reference is a Ruby/Python-shaped `Function` carrying the same + // five facts, built here rather than reused from `fixture()` (whose + // function has no writes or calls, so it could only prove the tiers + // agree on the two predicates it happens to emit). + let mut fn_graph = ModelGraph::new("ruby"); + let mut fn_model = Model::new("Order"); + fn_model.functions.push(Function { + name: "settle".to_string(), + reads: vec!["total".to_string()], + raises: vec!["UserError".to_string()], + writes: vec!["state".to_string()], + guarded_writes: vec!["state".to_string()], + calls: vec!["self.save".to_string()], + ..Function::default() + }); + fn_graph.models.push(fn_model); + let fn_triples = expand(&fn_graph); + let fn_truth = |p: &str| fn_triples.iter().find(|t| t.p == p).map(|t| (t.f, t.c)); + for p in [ + "reads_field", + "writes_field", + "raises", + "calls", + "writes_if_blank", + ] { + let cpp = truth(p); + assert!(cpp.is_some(), "C++ fixture emitted no `{p}`"); + assert_eq!( + cpp, + fn_truth(p), + "`{p}` tier differs between C++ and Function paths" + ); + } + } + + /// The C++ body arm expands to the SAME five predicates, with the SAME + /// object encoding, as a Ruby/Python `Function` body: own members join + /// the `cpp_field` node, exceptions take the `exc:` namespace, calls stay + /// verbatim `receiver.method` strings. Anchored on the per-overload + /// method IRI so a body fact can never attach to the wrong overload. + #[test] + fn cpp_method_body_arm_expands_to_the_function_quartet() { + let triples = expand(&cpp_fixture()); + let m = "cpp:Tesseract::Recognizer.Recognize(int,const Image &) const"; + let has = + |s: &str, p: &str, o: &str| triples.iter().any(|t| t.s == s && t.p == p && t.o == o); + assert!(has(m, "writes_field", "cpp:Tesseract::Recognizer.status_")); + assert!(has( + m, + "writes_field", + "cpp:Tesseract::Recognizer.recognizer_" + )); + assert!(has( + m, + "reads_field", + "cpp:Tesseract::Recognizer.recognizer_" + )); + assert!(has( + m, + "writes_if_blank", + "cpp:Tesseract::Recognizer.recognizer_" + )); + assert!(has(m, "raises", "exc:BadStatus")); + assert!(has(m, "calls", "repo_.Save")); + // The written member joins the field node `cpp_field` emitted. + assert!(has( + "cpp:Tesseract::Recognizer", + "has_field", + "cpp:Tesseract::Recognizer.recognizer_" + )); + // Silence half: the three arm-less overloads emit no body facts at all. + for t in &triples { + if matches!( + t.p.as_str(), + "writes_field" | "reads_field" | "raises" | "calls" | "writes_if_blank" + ) { + assert_eq!(t.s, m, "body fact `{}` leaked onto `{}`", t.p, t.s); + } + } } /// Every C++ machine-plane predicate fires from the fixture — the C++ @@ -2575,6 +2722,11 @@ mod tests { "is_const", "is_static", "has_visibility", + "writes_field", + "reads_field", + "raises", + "calls", + "writes_if_blank", ] { assert!( seen.contains(p), diff --git a/crates/ruff_spo_triplet/src/ir.rs b/crates/ruff_spo_triplet/src/ir.rs index 9cff29b5b5ee49..473f73ea7a6f53 100644 --- a/crates/ruff_spo_triplet/src/ir.rs +++ b/crates/ruff_spo_triplet/src/ir.rs @@ -679,12 +679,25 @@ pub struct CppField { pub type_name: String, } -/// One method declaration carrying its C++ property flags. +/// One method declaration carrying its C++ property flags — and, when the +/// harvester saw the body, its **body arm**. /// /// Every method is classified (`rdf:type Function` + `has_function`); each /// set flag additionally expands to a method-property predicate. The flags /// are not mutually exclusive (a method can be both `constexpr` and /// `noexcept`, an `operator` and an `override`). +/// +/// The five trailing `Vec` fields ([`Self::writes`] / [`Self::reads`] / +/// [`Self::raises`] / [`Self::calls`] / [`Self::guarded_writes`]) are the +/// C++ reading of the [`Function`] body quartet — the `(W, R, X, C)` +/// fingerprint plus the J1 guard the recipe codebook +/// (`.claude/knowledge/fuzzy-recipe-codebook.md`) classifies on. They expand +/// to the SAME five predicates with the SAME object encoding and truth tiers +/// as the Ruby/Python `Function` path, so `recipe::classify` and the OGAR +/// `lift_actions` feed see one shape regardless of frontend. All five are +/// `skip_serializing_if`-empty: a signature-only harvest (bodies skipped, or +/// a declaration whose definition lives in a TU the walker never saw) leaves +/// the ndjson byte-identical to the pre-arm shape. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[expect( clippy::struct_excessive_bools, @@ -747,6 +760,38 @@ pub struct CppMethod { /// likely internal). Defaults to `Public`. #[serde(default)] pub access: CppAccess, + /// Own data members the body ASSIGNS (`x_ = v`, `x_ += v`, `++x_`, + /// `x_.operator=(v)`, and a `this->x_` spelling of each). Emitted as + /// `writes_field` (Authoritative — the assignment names its target). + /// Members of OTHER objects (`other.x_ = v`) are deliberately not + /// writes: the fingerprint is about *this* class's state. Sorted, deduped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub writes: Vec, + /// Own data members the body READS (any member reference that is not the + /// bare assignment target; compound assignment and `++` count as both a + /// read and a write). Emitted as `reads_field` (Inferred). Sorted, deduped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub reads: Vec, + /// Exception type names the body `throw`s, bare (`BadStatus`, not + /// `exc:BadStatus` — the expander adds the `exc:` namespace exactly as it + /// does for [`Function::raises`]). Emitted as `raises` (Authoritative). + /// Sorted, deduped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub raises: Vec, + /// Lifecycle-mutator calls the body dispatches, as `"."` + /// (`repo_.Save`, `this.Flush`). Only the configured mutator set is + /// captured, mirroring [`Function::calls`] — the signal is "this method + /// calls a writer", not a full call graph. Emitted as `calls` (Inferred). + /// Sorted, deduped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub calls: Vec, + /// The J1 fact: own members whose write sits under an absence test on + /// that same member (`if (x_ == nullptr) x_ = …;`, `if (!x_) …`, + /// `if (x_.empty()) …`, or the else-branch of the presence form). Always a + /// subset of [`Self::writes`]; emitted as `writes_if_blank` + /// (Authoritative). Sorted, deduped. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub guarded_writes: Vec, } /// `constexpr` vs `consteval` compile-time markers. @@ -872,6 +917,31 @@ mod dto_surface_tests { "writes", ]; + /// The full `CppMethod` DTO surface, pinned. The five body-arm keys + /// (`calls`/`guarded_writes`/`raises`/`reads`/`writes`) are the C++ + /// reading of the [`FUNCTION_DTO`] quartet + J1 fact and share its + /// consumers (`recipe::classify`, `ogar-from-ruff::lift_actions`); the + /// rest is the signature plane `ruff_cpp_codegen` reconstructs from. + const CPP_METHOD_DTO: &[&str] = &[ + "access", + "calls", + "constexpr_kind", + "guarded_writes", + "is_const", + "is_noexcept", + "is_pure_virtual", + "is_static", + "name", + "operator_kind", + "overrides", + "param_types", + "raises", + "reads", + "requires_clause", + "return_type", + "writes", + ]; + /// The full `Field` DTO surface, pinned. const FIELD_DTO: &[&str] = &[ "depends_on", @@ -921,6 +991,28 @@ mod dto_surface_tests { "validations", ]; + fn full_cpp_method() -> CppMethod { + CppMethod { + name: "m".into(), + is_pure_virtual: true, + constexpr_kind: Some(ConstexprKind::Constexpr), + is_noexcept: true, + overrides: Some("Base::m".into()), + operator_kind: Some("==".into()), + requires_clause: Some("C".into()), + return_type: Some("int".into()), + param_types: vec!["int".into()], + is_const: true, + is_static: true, + access: CppAccess::Public, + writes: vec!["w_".into()], + reads: vec!["r_".into()], + raises: vec!["E".into()], + calls: vec!["repo_.Save".into()], + guarded_writes: vec!["w_".into()], + } + } + fn full_function() -> Function { Function { name: "f".into(), @@ -1024,20 +1116,7 @@ mod dto_surface_tests { name: "m".into(), type_name: "int".into(), }], - methods: vec![CppMethod { - name: "m".into(), - is_pure_virtual: true, - constexpr_kind: Some(ConstexprKind::Constexpr), - is_noexcept: true, - overrides: Some("Base::m".into()), - operator_kind: Some("==".into()), - requires_clause: Some("C".into()), - return_type: Some("int".into()), - param_types: vec!["int".into()], - is_const: true, - is_static: true, - access: CppAccess::Public, - }], + methods: vec![full_cpp_method()], templates: vec![CppTemplate { kind: CppTemplateKind::Specialisation, name: "T".into(), @@ -1068,6 +1147,25 @@ mod dto_surface_tests { assert_eq!(keys(&full_model()), MODEL_DTO); } + #[test] + fn cpp_method_dto_surface_locked() { + assert_eq!(keys(&full_cpp_method()), CPP_METHOD_DTO); + } + + /// The C++ body arm is the `Function` quartet + J1 fact under the SAME + /// key names, so one recipe classifier / one OGAR lift serves both + /// frontends. A rename on one side must bang here before it ships. + #[test] + fn cpp_method_body_arm_keys_are_the_function_body_keys() { + for k in ["calls", "guarded_writes", "raises", "reads", "writes"] { + assert!(FUNCTION_DTO.contains(&k), "`{k}` missing from FUNCTION_DTO"); + assert!( + CPP_METHOD_DTO.contains(&k), + "`{k}` missing from CPP_METHOD_DTO" + ); + } + } + #[test] fn actiondef_feed_is_subset_of_function_dto() { for k in ACTIONDEF_FEED_FUNCTION { diff --git a/crates/ruff_spo_triplet/src/lib.rs b/crates/ruff_spo_triplet/src/lib.rs index 084bbe930c6336..408dc4171fb10e 100644 --- a/crates/ruff_spo_triplet/src/lib.rs +++ b/crates/ruff_spo_triplet/src/lib.rs @@ -88,7 +88,7 @@ pub use nav_digest::build_nav_digest; pub use ndjson::{ParseError, from_ndjson, to_ndjson}; pub use quad::{MenuQuad, PurposeRole, PurposeRule, classify_purpose}; pub use reassemble::{cpp_projection, reassemble}; -pub use recipe::{RecipeCentroid, classify, is_recoverable}; +pub use recipe::{BodyFacts, RecipeCentroid, classify, is_recoverable}; pub use region::{RegionFact, RegionSubject, region_triples}; pub use structured_names::{ NameGrammar, StructuredName, Tier, parse_structured_name, part_of_edges, diff --git a/crates/ruff_spo_triplet/src/reassemble.rs b/crates/ruff_spo_triplet/src/reassemble.rs index ebed7dfee4286c..6e6afaf5a9aba0 100644 --- a/crates/ruff_spo_triplet/src/reassemble.rs +++ b/crates/ruff_spo_triplet/src/reassemble.rs @@ -75,6 +75,14 @@ struct MethodAcc { is_const: bool, is_static: bool, access: CppAccess, + /// Body arm — recovered from the same five predicates `expand` emits for + /// a `Function` body (`writes_field` / `reads_field` / `raises` / `calls` + /// / `writes_if_blank`), objects stripped back to the bare IR spelling. + writes: Vec, + reads: Vec, + raises: Vec, + calls: Vec, + guarded_writes: Vec, } /// Reassemble the C++ machine-plane projection of a triple set into a @@ -192,6 +200,35 @@ pub fn reassemble(triples: &[Triple]) -> ModelGraph { }; } } + // Body arm. Own-member objects are `{class_iri}.{member}` — the + // owner comes from the `has_function` seed, never from splitting + // the method IRI (anchor-first attribution, same as everything + // else here). `raises` drops the `exc:` namespace `expand` added; + // `calls` is a verbatim `"receiver.method"` string on both sides. + "writes_field" | "reads_field" | "writes_if_blank" => { + if let Some(class_iri) = method_owner.get(&t.s) + && let Some(acc) = method_acc.get_mut(&t.s) + { + let prefix = format!("{class_iri}."); + let member = t.o.strip_prefix(&prefix).unwrap_or(&t.o).to_string(); + match t.p.as_str() { + "writes_field" => acc.writes.push(member), + "reads_field" => acc.reads.push(member), + _ => acc.guarded_writes.push(member), + } + } + } + "raises" => { + if let Some(acc) = method_acc.get_mut(&t.s) { + let exc = t.o.strip_prefix("exc:").unwrap_or(&t.o).to_string(); + acc.raises.push(exc); + } + } + "calls" => { + if let Some(acc) = method_acc.get_mut(&t.s) { + acc.calls.push(t.o.clone()); + } + } "has_field" => { if let Some(model) = classes.get_mut(&t.s) { let prefix = format!("{}.", t.s); @@ -293,6 +330,11 @@ pub fn reassemble(triples: &[Triple]) -> ModelGraph { is_const: acc.is_const, is_static: acc.is_static, access: acc.access, + writes: acc.writes, + reads: acc.reads, + raises: acc.raises, + calls: acc.calls, + guarded_writes: acc.guarded_writes, }); } @@ -354,6 +396,21 @@ fn canonicalize_cpp(graph: &mut ModelGraph) { // must sort deterministically (non-const before const) on both the // reassembled and the projected side; without it the stable sort // preserves two different pre-sort orders and the pair compares unequal. + // The body arm is a set of facts on both sides (the harvester sorts + + // dedups; `expand` dedups `(s, p, o)`), so canonicalise each Vec + // before the method-level sort/dedup compares whole methods. + for method in &mut model.methods { + for arm in [ + &mut method.writes, + &mut method.reads, + &mut method.raises, + &mut method.calls, + &mut method.guarded_writes, + ] { + arm.sort_unstable(); + arm.dedup(); + } + } model.methods.sort_by(|a, b| { (a.name.as_str(), &a.param_types, a.is_const).cmp(&( b.name.as_str(), @@ -431,6 +488,17 @@ mod tests { is_const: true, is_static: false, access: CppAccess::Public, + // Deliberately unsorted + duplicated: the projection must + // canonicalise the arm exactly as `expand`'s dedup does. + writes: vec![ + "status_".to_string(), + "recognizer_".to_string(), + "status_".to_string(), + ], + reads: vec!["recognizer_".to_string()], + raises: vec!["BadStatus".to_string()], + calls: vec!["repo_.Save".to_string()], + guarded_writes: vec!["recognizer_".to_string()], }); rec.methods.push(CppMethod { name: "Clear".to_string(), @@ -445,6 +513,11 @@ mod tests { is_const: false, is_static: false, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.methods.push(CppMethod { name: "kMaxRating".to_string(), @@ -459,6 +532,11 @@ mod tests { is_const: false, is_static: true, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.methods.push(CppMethod { name: "operator==".to_string(), @@ -473,6 +551,11 @@ mod tests { is_const: false, is_static: false, access: CppAccess::Public, + writes: Vec::new(), + reads: Vec::new(), + raises: Vec::new(), + calls: Vec::new(), + guarded_writes: Vec::new(), }); rec.templates.push(CppTemplate { kind: CppTemplateKind::Specialisation, @@ -508,6 +591,84 @@ mod tests { assert_eq!(got, projected(&g)); } + /// The body arm survives the round-trip on the per-overload node it was + /// harvested from: own members recover bare (the `{class}.` prefix + /// stripped), `raises` recovers bare (the `exc:` namespace stripped), + /// `calls` recovers verbatim, and a duplicated source fact collapses to + /// one — the same `(s, p, o)` dedup `expand` applies. The arm-less + /// overloads recover empty, so a body fact can never bleed across + /// overloads through the accumulator. + #[test] + fn body_arm_round_trips_on_its_own_overload() { + let g = cpp_fixture(); + let got = reassemble(&expand(&g)); + assert_eq!(got, projected(&g)); + + let rec = got + .models + .iter() + .find(|m| m.name == "Tesseract::Recognizer") + .unwrap(); + let recognize = rec.methods.iter().find(|m| m.name == "Recognize").unwrap(); + assert_eq!(recognize.writes, vec!["recognizer_", "status_"]); + assert_eq!(recognize.reads, vec!["recognizer_"]); + assert_eq!(recognize.raises, vec!["BadStatus"]); + assert_eq!(recognize.calls, vec!["repo_.Save"]); + assert_eq!(recognize.guarded_writes, vec!["recognizer_"]); + for other in rec.methods.iter().filter(|m| m.name != "Recognize") { + assert!(other.writes.is_empty(), "{} gained writes", other.name); + assert!(other.reads.is_empty(), "{} gained reads", other.name); + assert!(other.raises.is_empty(), "{} gained raises", other.name); + assert!(other.calls.is_empty(), "{} gained calls", other.name); + assert!( + other.guarded_writes.is_empty(), + "{} gained guarded_writes", + other.name + ); + } + } + + /// A const/non-const overload pair with DIFFERENT body arms must keep + /// each arm on its own node — the cv-qualified IRI is what makes the two + /// accumulators distinct, so this is the body-arm reading of + /// [`const_and_nonconst_overload_stay_distinct`]. + #[test] + fn body_arms_do_not_merge_across_a_const_overload_pair() { + let mut g = ModelGraph::new("cpp"); + let mut m = Model::new("GenericVector"); + m.methods.push(CppMethod { + name: "at".to_string(), + return_type: Some("T &".to_string()), + param_types: vec!["int".to_string()], + is_const: false, + writes: vec!["data_".to_string()], + reads: vec!["data_".to_string()], + ..Default::default() + }); + m.methods.push(CppMethod { + name: "at".to_string(), + return_type: Some("const T &".to_string()), + param_types: vec!["int".to_string()], + is_const: true, + reads: vec!["data_".to_string()], + ..Default::default() + }); + g.models.push(m); + + let got = reassemble(&expand(&g)); + assert_eq!(got, projected(&g)); + let methods = &got.models[0].methods; + assert_eq!(methods.len(), 2); + let non_const = methods.iter().find(|m| !m.is_const).unwrap(); + let const_ = methods.iter().find(|m| m.is_const).unwrap(); + assert_eq!(non_const.writes, vec!["data_"]); + assert!( + const_.writes.is_empty(), + "the const accessor gained a write" + ); + assert_eq!(const_.reads, vec!["data_"]); + } + /// Two classes that declare an identically-named, identically-signatured /// method (the real `UNICHARMAP::unichar_to_id` / /// `UNICHARSET::unichar_to_id` collision) must each keep their own diff --git a/crates/ruff_spo_triplet/src/recipe.rs b/crates/ruff_spo_triplet/src/recipe.rs index d43c28f3cfe668..726570874bd4f3 100644 --- a/crates/ruff_spo_triplet/src/recipe.rs +++ b/crates/ruff_spo_triplet/src/recipe.rs @@ -9,7 +9,67 @@ //! methods alike. A frontend "adds the arm" purely by populating those //! `Vec`s from its own AST; this module runs unchanged on the result. -use crate::ir::Function; +use crate::ir::{CppMethod, Function}; + +/// The body fact-set a centroid is computed from — the `(W, R, X, C)` +/// fingerprint plus the J1 guard, borrowed from whichever IR struct carries +/// it. +/// +/// This trait is what makes the module's claim ("the identical ladder +/// classifies Ruby hooks, Odoo `_compute_*` methods, C# handlers, and C++ +/// methods alike") true rather than aspirational: [`classify`] is written +/// once against these five slices, and a frontend joins by populating them. +/// Two IR structs implement it today — [`Function`] (the Ruby/Python/C# +/// plane) and [`CppMethod`] (the C++ machine plane) — and they classify +/// identically because the ladder never sees which one it was handed. +pub trait BodyFacts { + /// Fields the body assigns. + fn writes(&self) -> &[String]; + /// Fields the body reads. + fn reads(&self) -> &[String]; + /// Exception/error type names the body raises. + fn raises(&self) -> &[String]; + /// Lifecycle-mutator calls the body dispatches. + fn calls(&self) -> &[String]; + /// The J1 subset of [`Self::writes`] guarded by an absence test. + fn guarded_writes(&self) -> &[String]; +} + +impl BodyFacts for Function { + fn writes(&self) -> &[String] { + &self.writes + } + fn reads(&self) -> &[String] { + &self.reads + } + fn raises(&self) -> &[String] { + &self.raises + } + fn calls(&self) -> &[String] { + &self.calls + } + fn guarded_writes(&self) -> &[String] { + &self.guarded_writes + } +} + +impl BodyFacts for CppMethod { + fn writes(&self) -> &[String] { + &self.writes + } + fn reads(&self) -> &[String] { + &self.reads + } + fn raises(&self) -> &[String] { + &self.raises + } + fn calls(&self) -> &[String] { + &self.calls + } + fn guarded_writes(&self) -> &[String] { + &self.guarded_writes + } +} /// The nearest declarative recipe a method body correlates to, or one of /// the two irreducible "essential" kinds that stay hand-ported. @@ -55,11 +115,11 @@ pub enum RecipeCentroid { /// [`RecipeCentroid::Compute`] / [`RecipeCentroid::Normalize`] are checked, /// per the J1 finding recorded there. #[must_use] -pub fn classify(f: &Function) -> RecipeCentroid { - let writes = !f.writes.is_empty(); - let reads = !f.reads.is_empty(); - let raises = !f.raises.is_empty(); - let calls = !f.calls.is_empty(); +pub fn classify(f: &F) -> RecipeCentroid { + let writes = !f.writes().is_empty(); + let reads = !f.reads().is_empty(); + let raises = !f.raises().is_empty(); + let calls = !f.calls().is_empty(); if calls && raises { return RecipeCentroid::Compensate; @@ -73,13 +133,13 @@ pub fn classify(f: &Function) -> RecipeCentroid { if writes && raises { return RecipeCentroid::WriteRaise; } - if writes && is_subset(&f.writes, &f.guarded_writes) { + if writes && is_subset(f.writes(), f.guarded_writes()) { return RecipeCentroid::Default; } - if writes && !is_subset(&f.writes, &f.reads) { + if writes && !is_subset(f.writes(), f.reads()) { return RecipeCentroid::Compute; } - if writes && is_subset(&f.writes, &f.reads) { + if writes && is_subset(f.writes(), f.reads()) { return RecipeCentroid::Normalize; } if !writes && !raises && !calls && reads { @@ -243,4 +303,127 @@ mod tests { assert!(f.reads.is_empty()); assert_eq!(classify(&f), RecipeCentroid::Default); } + + /// The trait's whole claim, measured: a C++ method and a Ruby/Python + /// function carrying the SAME facts land on the SAME centroid, across + /// every branch of the ladder. A frontend joins the classifier by + /// populating five `Vec`s — it does not get its own ladder. + /// + /// The fact-sets below are chosen to reach each centroid in turn, so a + /// ladder that agreed only on the common cases could not pass. + #[test] + fn a_cpp_method_and_a_function_with_the_same_facts_classify_alike() { + /// One row of the ladder: the five fact-sets, and the centroid they + /// must reach on either frontend's struct. + struct Case { + writes: &'static [&'static str], + reads: &'static [&'static str], + raises: &'static [&'static str], + calls: &'static [&'static str], + guarded: &'static [&'static str], + expected: RecipeCentroid, + } + let cases = [ + Case { + writes: &["s"], + reads: &[], + raises: &["E"], + calls: &["r.save"], + guarded: &[], + expected: RecipeCentroid::Compensate, + }, + Case { + writes: &[], + reads: &[], + raises: &[], + calls: &["r.save"], + guarded: &[], + expected: RecipeCentroid::Cascade, + }, + Case { + writes: &[], + reads: &["s"], + raises: &["E"], + calls: &[], + guarded: &[], + expected: RecipeCentroid::Guard, + }, + Case { + writes: &["s"], + reads: &[], + raises: &["E"], + calls: &[], + guarded: &[], + expected: RecipeCentroid::WriteRaise, + }, + Case { + writes: &["s"], + reads: &[], + raises: &[], + calls: &[], + guarded: &["s"], + expected: RecipeCentroid::Default, + }, + Case { + writes: &["s"], + reads: &[], + raises: &[], + calls: &[], + guarded: &[], + expected: RecipeCentroid::Compute, + }, + Case { + writes: &["s"], + reads: &["s"], + raises: &[], + calls: &[], + guarded: &[], + expected: RecipeCentroid::Normalize, + }, + Case { + writes: &[], + reads: &["s"], + raises: &[], + calls: &[], + guarded: &[], + expected: RecipeCentroid::Observe, + }, + Case { + writes: &[], + reads: &[], + raises: &[], + calls: &[], + guarded: &[], + expected: RecipeCentroid::Empty, + }, + ]; + let own = |v: &[&str]| v.iter().map(|s| (*s).to_string()).collect::>(); + for c in cases { + let function = Function { + name: "f".to_string(), + writes: own(c.writes), + reads: own(c.reads), + raises: own(c.raises), + calls: own(c.calls), + guarded_writes: own(c.guarded), + ..Function::default() + }; + let method = CppMethod { + name: "f".to_string(), + writes: own(c.writes), + reads: own(c.reads), + raises: own(c.raises), + calls: own(c.calls), + guarded_writes: own(c.guarded), + ..CppMethod::default() + }; + let want = c.expected; + assert_eq!(classify(&function), want, "Function should be {want:?}"); + assert_eq!( + classify(&method), + classify(&function), + "C++ should match {want:?}" + ); + } + } } From e95f4471e9eac9ca5ec29c0cbf7e5c46241d8ab6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:33 +0000 Subject: [PATCH 02/19] Correct four stale claims in the ruff_cpp_spo module doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each was true when written and is not now: * "the 13 C++ predicates" — a count in prose goes stale the first time a predicate is minted, and is then cited as evidence. Replaced with a pointer to the test that actually asserts the set. * "`extract` ... remains `todo!()` (per-TU include resolution + cross-TU dedup)" — the tree walk and the cross-TU dedup ship as `extract_dir` / `extract_tree`. Only per-TU include auto-detection is missing, which is the whole of what `extract` would add. * "behind a `libclang` feature, when wired" — it is wired. * `predicate_count_locked_at_47` names a gate that does not exist; the gate is `predicate_count_locked_at_79` in `triple.rs`. A reader grepping for the old name found nothing. The last one now says the number moves with the vocabulary and the gate is the test, so the reference cannot rot the same way again. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/src/lib.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/ruff_cpp_spo/src/lib.rs b/crates/ruff_cpp_spo/src/lib.rs index 8ed377ce0d332e..4d5108d7cca4c6 100644 --- a/crates/ruff_cpp_spo/src/lib.rs +++ b/crates/ruff_cpp_spo/src/lib.rs @@ -4,8 +4,11 @@ //! Walks a C++ corpus (Tesseract first; LLVM / Boost / `OpenCV` next) via //! libclang and produces a [`ModelGraph`] populated with the C++ machine- //! plane `Declaration` siblings the shared `ruff_spo_triplet` crate expands -//! into the 13 C++ predicates (`inherits_from`, `template_specialises`, -//! `virtually_overrides`, `is_pure_virtual`, …). +//! into the C++ machine-plane predicates (`inherits_from`, +//! `template_specialises`, `virtually_overrides`, `is_pure_virtual`, …). +//! The exact set is whatever `expand::tests::cpp_emits_every_cpp_predicate` +//! asserts — deliberately not restated here, because a count in prose goes +//! stale the first time a predicate is minted and is then cited as evidence. //! //! # The harvester family //! @@ -32,21 +35,26 @@ //! semantic transform, no re-parsing. //! - `walk_tu` (feature `libclang`) walks ONE translation unit via real //! libclang and returns [`CppClass`] definitions (classes/bases/fields/ -//! methods with their flags, system-header classes filtered out). -//! [`extract`] — the corpus-TREE orchestration over `walk_tu` — remains -//! `todo!()` (per-TU include resolution + cross-TU dedup). The target -//! triple shape is locked by `tests::locked_shape_expands_to_expected_triples`. +//! methods with their flags and body arm, system-header classes filtered +//! out). `extract_dir` / `extract_tree` walk a directory and a whole tree, +//! dedup classes by qualified name across translation units, and fold each +//! method's body arm in from whichever unit had the definition. [`extract`] +//! is still `todo!()`: what it adds over `extract_tree` is per-TU include +//! auto-detection, not the tree walk or the dedup. The target triple shape +//! is locked by `tests::locked_shape_expands_to_expected_triples`. //! //! # Iron rules this frontend respects //! //! - **`ruff_spo_triplet` stays serde-only.** The libclang dependency lives -//! here (behind a `libclang` feature, when wired), never in the shared -//! core. +//! here, behind the `libclang` feature, never in the shared core. //! - **No C++ source vendored into a `*-rs` target.** The corpus stays //! upstream; `extract` walks it from a configurable path. //! - **Closed-vocab gate.** The C++ predicates are in -//! `ruff_spo_triplet::Predicate` under the `predicate_count_locked_at_47` -//! gate. A new C++ predicate is a deliberate ontology change there. +//! `ruff_spo_triplet::Predicate` under the `predicate_count_locked_at_79` +//! gate in `triple.rs` (the number moves with the vocabulary; the gate is +//! the test). A new C++ predicate is a deliberate ontology change there — +//! the method body arm added none, because it reuses the five predicates +//! the Ruby/Python `Function` body already emits. use std::path::Path; From b895ba51f620e52b9f251f65eab69753657532cb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:33 +0000 Subject: [PATCH 03/19] Pin the two nested-if branches of the J1 guard threading `if_stmt` decides a guard per branch, and two of its outcomes had no test: an enclosing absence guard crossing a nested `if` that has no guard of its own (the write is still only reachable when the member was absent), and that same guard being dropped by a nested `if` that guards on a different member. Both halves are disable-verified. Threading `None` instead of the enclosing guard fails the first; letting `record_write` accept any guard rather than one naming the written member fails the second, and takes the existing other-member case down with it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/src/clang_walker.rs | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index d3039fa1d24452..7de1dd60d1ef5f 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -1583,6 +1583,8 @@ struct Cfg { void wrong_branch(int* v) { if (ptr_ == nullptr) { } else { ptr_ = v; } } void other_member_guard(int* v) { if (other_ == 0) { ptr_ = v; } } void compound_condition(int* v, bool b) { if (ptr_ == nullptr && b) { ptr_ = v; } } + void nested_plain_if(int* v, bool b) { if (ptr_ == nullptr) { if (b) { ptr_ = v; } } } + void nested_regard(int* v) { if (ptr_ == nullptr) { if (other_ == 0) { ptr_ = v; } } } }; "#; @@ -1642,6 +1644,33 @@ struct Cfg { } } + /// An enclosing absence guard survives a nested `if` that has no guard of + /// its own — the write is still only reachable when the member was absent. + /// It does NOT survive a nested `if` that guards on a DIFFERENT member, + /// because that inner condition decides its branches and the outer guard + /// is dropped rather than reasoned about. Both are branches of the guard + /// threading that no other test reaches. + #[test] + fn an_enclosing_guard_crosses_a_plain_nested_if_but_not_a_regarding_one() { + let all = arms("guards", GUARDS); + + let crossed = &all["nested_plain_if"]; + assert_eq!(crossed.writes, ["ptr_"]); + assert_eq!( + crossed.guarded_writes, + ["ptr_"], + "a plain nested `if` does not cancel the enclosing absence guard" + ); + + let regarded = &all["nested_regard"]; + assert_eq!(regarded.writes, ["ptr_"]); + assert!( + regarded.guarded_writes.is_empty(), + "an inner guard on another member drops the outer one: {:?}", + regarded.guarded_writes + ); + } + #[test] fn the_mutator_vocabulary_is_configurable() { let src = r#" From ec4a48231d0837e0cb60734fcba780a5e88d1b49 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:33 +0000 Subject: [PATCH 04/19] Record the C++ body arm in the probe plan and the recipe codebook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codebook's coverage table had listed `ruff_cpp_spo` as scaffolded since 2026-07-06, and its "a frontend adds the arm by populating those Vecs — zero IR change" line was true only for the three frontends that route methods through `Function` itself. C++ has its own method struct, so wiring its arm cost a real (additive) IR change plus a trait, and the doc now says so. The probe plan gains a dated entry stating what is measured, what is not, and the next probe. Three of its claims are corrected here against facts the entry was written without: * it named a test (`provenance_tiers_match_ruby_reference`) that does not exist; the extended test is `cpp_truth_tiers_match_calibration`. * it said the feature had zero corpus-scale numbers. A one-off census has since been run and is recorded, with the finding that the same corpus parsed under C++17 rather than C++20 loses a third of its body facts to silently dropped declarations — the exact partial-AST failure the walker's own diagnostics warn about, reproducing on a real tree. * it flagged one of the two green disable runs as unidentifiable. It was the signature-only opt-out, which is enforced by two independent mechanisms, so no single disable can turn it red. `guarded_writes` reads 0 on that corpus and the entry says why: the whole tree holds three absence tests on a member and none of them is a write. That is a statement about the corpus, not about the detector. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/knowledge/fuzzy-recipe-codebook.md | 44 ++++- .claude/plans/cpp-spo-probes-v1.md | 206 +++++++++++++++++++++ 2 files changed, 247 insertions(+), 3 deletions(-) diff --git a/.claude/knowledge/fuzzy-recipe-codebook.md b/.claude/knowledge/fuzzy-recipe-codebook.md index 28614332e721f1..10f651a5e2fb02 100644 --- a/.claude/knowledge/fuzzy-recipe-codebook.md +++ b/.claude/knowledge/fuzzy-recipe-codebook.md @@ -52,7 +52,10 @@ on code: ## 2. The fingerprint — the DTO arm (this is what every frontend must emit) -The fingerprint is FOUR fact sets per method, on `ruff_spo_triplet::Function`: +The fingerprint is FOUR fact sets per method, on `ruff_spo_triplet::Function` +(and, since 2026-09-07, on `ruff_spo_triplet::CppMethod` too — see the dated +note beneath the coverage table below for why C++ needed its own copy of +these fields rather than reusing `Function`'s): | field | predicate emitted | provenance | what it captures | | ---------------- | ----------------- | ----------------- | --------------------------------------------------------------------------------------------------------------------- | @@ -71,6 +74,9 @@ landed.) > **This is the "DTO-arm shape" C++ / Python still need.** As of 2026-07-06 > `ruff_ruby_spo` and `ruff_csharp_spo` emit the full quartet + helpers. +> **[2026-09-07 correction: no longer true for C++ — the arm is now wired +> end-to-end, see the row below and the dated note that follows the table. +> Python is unaffected by this update and still needs it.]** > Coverage: > > | frontend | writes | reads | raises | calls | helpers | verdict | @@ -78,12 +84,24 @@ landed.) > | `ruff_ruby_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | reference — cook here first | > | `ruff_csharp_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | syntax-only (SemanticModel upgrade pending); helpers via `has_visibility`; tested end-to-end on a real production C# corpus (~97k triples) | > | `ruff_python_spo` | ~ | ✅ | ✅ | ~ | ✗ | reads/raises only; **needs writes/calls/helpers** | -> | `ruff_cpp_spo` | ~ | ~ | ~ | ~ | ✗ | scaffolded; **needs the arm populated** (setters/virtuals) | +> | `ruff_cpp_spo` | ✅ | ✅ | ✅ | ✅ | ✗ | **(2026-09-07)** wired end-to-end (commit `f068a3f`): harvested from real bodies via libclang 18, same predicates/objects/tiers as `Function`, classified via the now-generic `recipe::classify`; unit- and hermetic-fixture-proven (16 guards, disable-verified) — a one-off corpus census exists (ladybug `src/catalog`, 2794 methods, 20.7% reaching a non-`Empty` centroid; see `.claude/plans/cpp-spo-probes-v1.md`) but **nothing re-runs it**, so C++ has a measurement and not yet a gate; J1 hit-rate is still open, because that corpus contains no lazy-init writes to hit (helpers column unchanged: this commit didn't touch it, and C++ doesn't split private methods into a separate pool the way Ruby does — see the dated note below) | > > The fingerprint predicates are already in the shared IR > (`ruff_spo_triplet::Function`) and `expand()` already emits them — a frontend > "adds the arm" purely by *populating* those Vecs from its AST. Zero IR change. > Do it per-frontend, then this codebook runs unchanged on that language. +> **[2026-09-07 correction: "zero IR change" held for Ruby/Python/C# only +> because those three frontends already route their methods through +> `ruff_spo_triplet::Function` itself — there was nothing to add a field to +> but the one struct all three share. `ruff_cpp_spo` does NOT share it: its +> methods are a distinct IR struct, `CppMethod`, so wiring its arm required a +> real (additive) IR change — the same five fields added a second time, by +> name, on `CppMethod` (`ir.rs`) — plus a new `BodyFacts` trait so +> `recipe::classify` could stay a single function generic over either struct +> instead of forking into a `classify_function`/`classify_cpp_method` pair. +> Read "populate the Vecs, zero IR change" as the fast path for a frontend +> that already reuses `Function`; a frontend with its own method type pays a +> one-time IR/trait-impl cost instead, then joins the same classifier.]** > > **`ruff_csharp_spo` also needed a configurability step the reference > frontend didn't**: Ruby's `AR_MUTATORS` is a closed, ORM-shaped set, but a @@ -92,7 +110,27 @@ landed.) > `--mutator-prefixes`/`--mutator-receivers` generalise the mutator predicate > from "closed name set" to "name set OR prefix, optionally receiver-scoped" > — the same recipe centroids then classify EF Core (`SaveChanges`) and -> bespoke ADO.NET (`main.mysql.add_x`) call sites identically. +> bespoke ADO.NET (`main.mysql.add_x`) call sites identically. **`ruff_cpp_spo` +> got the same generalization on 2026-09-07**: `BodyArmConfig::with_mutators`/ +> `with_mutator_prefixes` replace the shipped closed set (`Save`/`Update`/ +> `Commit`/…) for a corpus with its own persistence vocabulary — unit-tested, +> not yet measured against a real non-STL-shaped C++ DAL. +> +> **On the `helpers` column for C++, specifically:** unlike Ruby (which keeps +> routable actions and private hook targets in two separate `Model` slots, +> `functions` vs `helpers`) or C# (one slot, `has_visibility` distinguishes), +> `ruff_cpp_spo` has always kept EVERY method — public, protected, and +> private alike — in the ONE `CppMethod`/`methods` collection, carrying its +> own `access: CppAccess` field and already emitting a `has_visibility` +> predicate before this commit. So the "a frontend without helpers loses +> ~80% of its hooks to no-facts" failure mode this doc opens with may simply +> not apply to C++ in the same shape — a private method's body facts were +> never at risk of being dropped the way Ruby's private defs originally were. +> This is an observation about the existing (pre-2026-09-07) shape of the +> walker, not something this commit changed or measured; the ✗ is left as-is +> above rather than reinterpreted, and a real corpus run (the next probe, +> `.claude/plans/cpp-spo-probes-v1.md`) would be the way to confirm private +> C++ methods actually resolve as hook targets in practice. ## 3. The recipe codebook — the centroids (pure fact-set predicates, GENERIC) diff --git a/.claude/plans/cpp-spo-probes-v1.md b/.claude/plans/cpp-spo-probes-v1.md index a3e348c8f7b183..c1913ee244b5da 100644 --- a/.claude/plans/cpp-spo-probes-v1.md +++ b/.claude/plans/cpp-spo-probes-v1.md @@ -523,3 +523,209 @@ adapter known-gap remains. In-crate falsifier gains - **Test count:** **58 ruff_spo_triplet + 15 ruff_cpp_spo**, all green. `clippy -D warnings` + fmt clean on both crates. + +## Update — 2026-09-07 (body-arm harvest wired end-to-end — the recipe-codebook's fifth arm lands; no corpus census yet) + +Commit `f068a3f` on this branch ("Harvest C++ method bodies into the recipe +fingerprint") is a **new capability landing on the same walker these three +probes gate**, not a new primary probe of its own — recorded here rather than +in a new file. It closes the gap `.claude/knowledge/fuzzy-recipe-codebook.md` +§2's coverage table has flagged since 2026-07-06: `ruff_cpp_spo` was the one +frontend whose `method_body_arm` walker existed as a reviewed-but-dead-code +draft (`#[allow(dead_code)] // TESTED via arm_tests; wired into CppMethod+expand +is the follow-up`) — nothing on `CppMethod` could carry the fingerprint the +codebook classifies on (`writes`/`reads`/`raises`/`calls` + the J1 +`guarded_writes` guard), and no `expand` arm could emit it. + +**What shipped (verified against the diff, file by file):** + +- `CppMethod` (`ruff_spo_triplet::ir`) gained the five body fields, under the + identical names, predicate mapping (`writes_field`/`reads_field`/`raises`/ + `calls`/`writes_if_blank`), object encoding, and truth tiers (Authoritative + for `writes_field`/`raises`/`writes_if_blank`, Inferred for + `reads_field`/`calls`) as the Ruby/C# `Function` quartet. `expand`'s + `cpp_method` arm emits all five onto the per-overload method IRI; `reassemble` + recovers them (own-member objects stripped back to the bare member name, + `raises` stripped of the `exc:` namespace `expand` adds); `canonicalize_cpp` + sorts+dedups each Vec before the method-level round-trip compare. A new + cross-frontend test, `expand::tests::cpp_method_body_arm_expands_to_the_function_quartet`, + asserts the exact predicate/object shape on a fixture and a silence half (no + arm-less overload gains a body fact); `expand::tests::cpp_truth_tiers_match_calibration` + is extended to assert the C++ tiers match a hand-built `Function` carrying the + same five facts, fact-for-fact. (Correction, same day: an earlier draft of + this entry named `provenance_tiers_match_ruby_reference`; no test of that + name exists in the crate.) +- `recipe::classify` — previously typed `fn classify(f: &Function)`, even + though its own doc comment already claimed to classify "Ruby hooks, Odoo + `_compute_*` methods, C# handlers, and C++ methods alike" — is now + `fn classify(f: &F)`, generic over a new `BodyFacts` + trait (`writes`/`reads`/`raises`/`calls`/`guarded_writes` accessors) + implemented for both `Function` and `CppMethod`. A new test, + `a_cpp_method_and_a_function_with_the_same_facts_classify_alike`, walks all + nine centroids (`Compensate`/`Cascade`/`Guard`/`WriteRaise`/`Default`/ + `Compute`/`Normalize`/`Observe`/`Empty`) and asserts a `CppMethod` and a + `Function` carrying the identical fact-set land on the identical centroid + every time — so the doc's cross-frontend claim is measured, not aspirational. +- `CppMethod` joined the DTO-surface lock as `CPP_METHOD_DTO`, and + `cpp_method_body_arm_keys_are_the_function_body_keys` asserts the five + body-arm keys sit in BOTH `FUNCTION_DTO` and `CPP_METHOD_DTO` — a rename on + one side now fails a test on the other before it ships. +- The walker (`clang_walker.rs`) was corrected against three defects the + earlier draft's own status comment admitted it could not have caught without + a real TU: (1) libclang exposes no binary-operator KIND on stable, so the + draft recorded the LHS of ANY binary operator (`if (status_ == v)`) as a + write — the operator is now read off the token stream + (`binary_operator_spelling`); (2) the assignment target also landed in + `reads`, so a pure setter classified `Normalize` instead of `Compute`; (3) + `guarded_writes` was never populated at all, so J1 could not fire on C++. + Two further shapes were found only by dumping real cursors, not by reading + the AST docs: a class-typed member assigns through `operator=` (arrives as a + `CallExpr`, not a `BinaryOperator`), and a call's own callee reference is a + `MemberRefExpr` that would otherwise misread as a read of a field named + after the method. +- `walk_tu` now parses bodies by default (`skip_function_bodies(false)`); + `walk_tu_configured(path, args, None)` opts back out for a faster + signature-only walk (arm left empty, signature plane byte-identical either + way — asserted by `a_signature_only_walk_leaves_the_arm_empty`). + `BodyArmConfig::{with_mutators, with_mutator_prefixes}` makes the + lifecycle-mutator vocabulary corpus-configurable, mirroring the C# + `--mutator-prefixes` generalization the codebook doc §2 already documents + (tested by `the_mutator_vocabulary_is_configurable`). +- The arm is harvested from `Entity::get_definition()`, not the declaration + cursor — required because a method declared in a header and defined + out-of-line in a `.cpp` has its body on a DIFFERENT cursor + (`an_out_of_line_definition_is_harvested_through_the_declaration` pins this). + `walk_files` (`ruff_cpp_spo::lib`) now merges body arms across translation + units via a new `merge_body_arms`: first-wins stays right for the signature + plane (every sighting of a declaration agrees), but the header's sighting of + an out-of-line method is arm-less, so an already-inserted empty arm is + filled from a later TU that has one — keyed on `(name, param_types, + is_const)` so a merge can never move one overload's body onto another, and + never overwriting an already-non-empty arm (so the result does not depend on + TU visit order). + +**MEASURED, not merely reviewed:** + +- Every cursor shape the walker matches (`x_ = v`, `p.x_ = v`, + `this->x_ = v`, `x_ += v`, `++x_`, `arr_[i] = v`, `repo_.Save()`, + `name_ = s` via `operator=`, `throw E()`) is documented in `clang_walker.rs` + as measured against **real libclang 18** on a fixture built for each + construct — the module comment says so explicitly ("Every cursor shape + matched below was measured against libclang 18 on a fixture carrying each + construct, not inferred from the AST documentation"). This checkout has + `libclang-18.so` present (`/usr/lib/llvm-18/lib`), consistent with the + `#[cfg(feature = "libclang")]`-gated tests actually running here rather than + being reviewed-only. +- `#[test]` counts, verified with `git show : | grep -c '#\[test\]'` + at the commit and its parent (a static source count, NOT a `cargo + test`/`nextest` run — I did not run cargo, per this session's constraints): + `ruff_cpp_spo/src/*` goes **23 → 41** (`clang_walker.rs` alone 6 → 23; + `lib.rs` 17 → 18 for the new cross-TU merge test); `ruff_spo_triplet/src/*` + goes **170 → 176** (`ir.rs` +2, `expand.rs` +1, `reassemble.rs` +2, + `recipe.rs` +1). Note the plan's own last "Test count" line above (58 + 15, + from 2026-06-17) is now far stale relative to the parent commit's actual + 170 / 23 — other, unrelated commits landed a great many tests on both + crates between 2026-06-17 and today; that drift predates and is unrelated + to this session's change. +- The commit message additionally claims "every guard is disable-verified + red-then-green (16 checks)." I did not independently re-run the + disable/re-enable exercise (that needs `cargo test`, which I did not run), + so I cannot confirm the count of 16 myself — I can only report that the + commit states it, and that one specific instance is corroborated by a code + comment (below). The 16 is a claim about *behavioral guards*, not the same + number as the 24 new `#[test]` functions above (a guard and a test are not + 1:1 — several of the new tests share one guard, and some pre-existing tests + were extended with new assertions against an already-counted `#[test]`). +- Two of those 16 disable runs came back green on the first pass and were + investigated rather than accepted, per the commit message. **One is fully + explained**, in both the commit message and a code comment on + `a_body_defined_out_of_line_survives_the_cross_tu_dedup` + (`crates/ruff_cpp_spo/src/lib.rs`): the cross-TU fixture's header, parsed + without `-x c++`, was silently interpreted as **C** by libclang's + extension-based language guess, so the header's `struct Svc` failed to + parse as a C++ class at all — the header contributed nothing, the `.cpp`'s + sighting was the ONLY sighting, and first-wins (i.e. no merge at all) + looked like a working merge. Fixed by adding `-x c++` to the test's `args` + and an up-front assertion (a direct `walk_tu_with_diagnostics` call on the + header alone) that the header's own sighting really is arm-less, so the + test cannot pass unless there are genuinely two different sightings to + merge. The other was **a mis-aimed disable, not a + weak test**, and is identified here because the commit message left it + abstract: the target was `a_signature_only_walk_leaves_the_arm_empty`. That + opt-out is enforced TWICE — the TU is parsed with `skip_function_bodies`, and + `build_method` does not harvest — so disabling either mechanism alone leaves + the behaviour correct and the test green. Forcing the parse flag on turns it + red, and so does forcing both. That is redundancy rather than vacuity, and it + is now recorded in a doc comment on the test itself, so a future reader who + wants to drop one of the two mechanisms as dead weight learns up front that + no single test will notice. + +**NOT yet measured — stated explicitly, not by omission:** + +- **Correction, same day** — an earlier draft of this entry said the feature + had "zero corpus-scale numbers". A one-off census HAS since been run, and + its numbers are below; what remains missing is a *committed, re-runnable* + census, because `crates/ruff_cpp_spo/examples/harvest_ladybug.rs` reports + only the signature plane (`inherits_from` trees, `has_function` top-25, + pure-virtual counts, `virtually_overrides`, field/template/static-assert + counts) and never calls `recipe::classify`. So the numbers below are a + measurement, not a gate: nothing re-runs them, and nothing fails if they + regress. + + `extract_tree` over `ladybug/src/catalog` (358 classes), every `CppMethod` + through `recipe::classify`: + + | | `-std=c++17` | `-std=c++20` | + | --- | ---: | ---: | + | methods | 2782 | 2794 | + | non-`Empty` centroid | 362 (13.0%) | 577 (20.7%) | + | writes / reads / raises / calls | 110 / 373 / 13 / 8 | 126 / 692 / 23 / 28 | + | census | `Observe=253 Compute=67 Normalize=21 Guard=13 Cascade=8` | `Observe=431 Compute=74 Cascade=25 Normalize=24 Guard=23` | + + **The two columns are the same corpus, one of them parsed wrongly, and the + gap is the finding.** `common/assert.h` needs `std::format`, so under + `-std=c++17` libclang emits an error diagnostic and recovers by silently + dropping declarations — costing a third of the body facts. That is exactly + the partial-AST failure mode `walk_tu_with_diagnostics`'s own doc comment + warns about ("0 failed from `walk_tu` alone does NOT mean the sweep is + complete"), reproducing on a real tree rather than in a fixture. Any future + corpus run must pass the standard the corpus actually needs and must read + the diagnostics, or it will quietly under-count. + + `guarded_writes` is **0** across all 2794 methods, and that is the corpus + rather than the detector: a search of the whole `ladybug/src` tree finds + three absence tests on a member, and all three are early returns or an + assertion — none is a guarded write. So there is still no C++ analog of the + Redmine "SelfMap population 1 Default / 3 Normalize" J1 split, but the + reason is that this corpus does not use the lazy-init shape, not that the + detector failed to see it. A corpus that does use it is what the J1 hit-rate + question actually needs. + + Still genuinely absent: a Tesseract-side run comparable to the "~97k + triples" C# figure, and any coarse triage PASS/FAIL or irreducible-core + roll. +- The corpus-configurable mutator vocabulary + (`BodyArmConfig::with_mutators`/`with_mutator_prefixes`) is unit-tested but + has not been exercised against a real non-STL-shaped C++ persistence layer + the way the C# `--mutator-prefixes` generalization was measured against a + real production corpus. +- J1 (`guarded_writes`) fires only on a LOCAL absence-test-guarding-the-branch + shape (no dominator analysis — by design, matching the Ruby/C# discipline); + this is unit-tested but its real-corpus hit rate on C++ is unmeasured, so + there is no C++ analog yet of the Redmine "SelfMap population 1 Default / + 3 Normalize" measurement the codebook doc §3 reports. + +**Next probe — call it `CPP-RECIPE-CENSUS`.** The one-off numbers above are +the first half of it; what is missing is making them re-runnable and gated. +Run the body-arm harvest over a real corpus (Tesseract's already-pinned +`src/ccutil` / `src/ccstruct`, or the new `harvest_ladybug.rs` target) and +report the same shape of numbers the Ruby leg reported in +`fuzzy-recipe-codebook.md` §4: coarse triage PASS/FAIL, the recipe-centroid +histogram, the irreducible-core check (only `Compensate`/`WriteRaise` should +survive the roll), and the J1 Default-vs-Normalize split — mirroring the C# +leg's "tested end-to-end on a real production C# corpus (~97k triples)" line +in the codebook's coverage table. Until that runs, treat "C++ recipe +classification works" as **unit-proven and plumbed, not corpus-validated**. + +- **Test count (source `#[test]` attributes, not a `cargo test` run):** + **176 ruff_spo_triplet + 41 ruff_cpp_spo** as of this commit. From 2978c12f441db64b7bcc23556423d8f6b744c2de Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:34 +0000 Subject: [PATCH 05/19] Report the body arm and a recipe-centroid census in the ladybug harvest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The example predated the body arm and reported only the signature plane, so the one capability that had no corpus-scale numbers also had no way to produce them. It now prints the five fact totals, a `RecipeCentroid` census via `recipe::classify` (every variant including the zeroes), and a top-classes-by- body-fact list — the same shape of numbers the Ruby and C# legs already report. Two things the doc header now states rather than leaving a reader to infer: * it walks headers only, so most bodies are inline accessors and a low non-`Empty` fraction is the example's scope, not a harvest gap. * `guarded_writes` reads 0 on this corpus because the whole tree holds three absence tests on a member and none is a lazy-init write. That is the corpus, not the detector. The clang args move to `-std=c++20`, which this corpus needs: under c++17, `common/assert.h` fails on `std::format` and libclang recovers by silently dropping the declarations after it, measured at roughly a third of the body facts (13.0% against 20.7% of methods reaching a non-`Empty` centroid). That is the partial-AST failure the example's own diagnostic warning exists to catch, so it should not have been walking into it. Run on `src/include/catalog`: 2424 methods, 381 with at least one body fact, census `Observe=283 Compute=62 Normalize=19 Cascade=10 Guard=7`. The 381 agrees exactly with an independent count of body-predicate subjects in the emitted ndjson. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .../ruff_cpp_spo/examples/harvest_ladybug.rs | 127 +++++++++++++++++- 1 file changed, 123 insertions(+), 4 deletions(-) diff --git a/crates/ruff_cpp_spo/examples/harvest_ladybug.rs b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs index 4fbdce6c706386..ce76f34e60cf8a 100644 --- a/crates/ruff_cpp_spo/examples/harvest_ladybug.rs +++ b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs @@ -1,9 +1,9 @@ //! Harvest an arbitrary ladybug (KuzuDB-fork) header **subtree**'s C++ class //! manifest via libclang, recursively, and emit the SPO manifest plus a full //! breakdown (`inherits_from` trees, `has_function` top-25, pure-virtual -//! classes, `virtually_overrides` count, and field, template, and -//! static-assert counts) — the `ruff>OGAR` structure feeding the ladybug-rs -//! transcode. +//! classes, `virtually_overrides` count, field/template/static-assert +//! counts, and a body-arm recipe-centroid census) — the `ruff>OGAR` +//! structure feeding the ladybug-rs transcode. //! //! Walks every `*.h` header under `src/include/` (recursively) via //! [`walk_tu_with_diagnostics`] (not [`walk_tu`](ruff_cpp_spo::walk_tu)) and @@ -14,6 +14,21 @@ //! declaration that needed the missing header (see `examples/harvest_textord.rs`'s //! `src/viewer` note for the concrete failure mode this was found against). //! +//! Also reports the **body arm** — `writes` / `reads` / `raises` / `calls` / +//! `guarded_writes` fact totals plus a `RecipeCentroid` census via +//! `ruff_spo_triplet::classify` — the cross-frontend recipe-centroid +//! comparison the Ruby and C# legs already produce, now surfaced for C++. +//! This example walks HEADERS only (no `.cpp` translation units), so most +//! bodies it sees are inline accessors: expect the non-`Empty` fraction to +//! read low relative to a whole-tree harvest — that is this example's scope, +//! not a harvest gap. +//! +//! Expect `guarded_writes` to read **0** on this corpus, and read that as the +//! corpus rather than the detector: a search of the whole `ladybug/src` tree +//! finds three absence tests on a member, and all three are early returns or +//! an assertion — none is the lazy-init `if (!x_) x_ = …` shape the J1 fact +//! records. A corpus that uses that shape is what a J1 hit-rate needs. +//! //! Env: //! `LADYBUG_SRC` default /home/user/ladybug //! `SUBTREE` relative to src/include, e.g. "storage" or "processor" (default "storage") @@ -24,6 +39,14 @@ //! LADYBUG_SRC=/home/user/ladybug LIBCLANG_PATH=/usr/lib/llvm-18/lib SUBTREE=storage \ //! cargo run -p ruff_cpp_spo --features libclang --example harvest_ladybug //! ``` +//! +//! This corpus needs `-std=c++20` (already the value baked into the clang +//! `args` below, not overridable via env): under `-std=c++17`, +//! `common/assert.h` fails to parse `std::format` and libclang silently +//! drops the declarations downstream of that failure — measured to cost +//! roughly a third of the body-arm facts (13.0% vs 20.7% of methods +//! reaching a non-`Empty` centroid). This is exactly the partial-AST +//! failure the `walk_tu_with_diagnostics` warning above exists to catch. #![expect( clippy::print_stderr, @@ -34,7 +57,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use ruff_cpp_spo::{CppClass, Declaration, NAMESPACE, model_from_class, walk_tu_with_diagnostics}; -use ruff_spo_triplet::{ModelGraph, expand, to_ndjson}; +use ruff_spo_triplet::{ModelGraph, RecipeCentroid, classify, expand, to_ndjson}; fn collect_headers(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { @@ -259,6 +282,102 @@ fn main() -> Result<(), Box> { } } + // ── body arm: writes / reads / raises / calls / guarded_writes ── + // This example walks HEADERS only (no `.cpp` translation units), so most + // bodies visible here are inline accessors — a low non-Empty fraction is + // this example's scope, not a harvest gap (see the module doc header). + let all_methods: Vec<&_> = all + .iter() + .flat_map(|c| c.declarations.iter()) + .filter_map(|d| match d { + Declaration::Method(m) => Some(m), + _ => None, + }) + .collect(); + let with_body_facts = all_methods + .iter() + .filter(|m| { + !m.writes.is_empty() + || !m.reads.is_empty() + || !m.raises.is_empty() + || !m.calls.is_empty() + || !m.guarded_writes.is_empty() + }) + .count(); + eprintln!( + "\n[{subtree}] body arm: {} methods, {with_body_facts} with at least one body fact", + all_methods.len(), + ); + + let n_writes: usize = all_methods.iter().map(|m| m.writes.len()).sum(); + let n_reads: usize = all_methods.iter().map(|m| m.reads.len()).sum(); + let n_raises: usize = all_methods.iter().map(|m| m.raises.len()).sum(); + let n_calls: usize = all_methods.iter().map(|m| m.calls.len()).sum(); + let n_guarded_writes: usize = all_methods.iter().map(|m| m.guarded_writes.len()).sum(); + eprintln!( + "[{subtree}] body-arm fact counts: writes={n_writes} reads={n_reads} raises={n_raises} calls={n_calls} guarded_writes={n_guarded_writes}" + ); + + // Recipe-centroid census — the headline: the same classifier the Ruby and + // C# legs already run against `Function`, here run against `CppMethod` + // via the shared `BodyFacts` trait (`ruff_spo_triplet::recipe`). + let centroids = [ + RecipeCentroid::Compensate, + RecipeCentroid::Cascade, + RecipeCentroid::Guard, + RecipeCentroid::WriteRaise, + RecipeCentroid::Default, + RecipeCentroid::Compute, + RecipeCentroid::Normalize, + RecipeCentroid::Observe, + RecipeCentroid::Empty, + ]; + let classified: Vec = all_methods.iter().map(|m| classify(*m)).collect(); + let mut census: Vec<(RecipeCentroid, usize)> = centroids + .iter() + .map(|&c| (c, classified.iter().filter(|&&x| x == c).count())) + .collect(); + census.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); + eprintln!("\n[{subtree}] recipe-centroid census (most common first):"); + for (centroid, n) in &census { + eprintln!(" {centroid:?}: {n}"); + } + + // top-10 classes by body-fact count (same shape as the template-decl list + // above — sum of writes/reads/raises/calls/guarded_writes per class). + let mut body_fact_counts: Vec<(String, usize)> = all + .iter() + .map(|c| { + let methods: Vec<&_> = c + .declarations + .iter() + .filter_map(|d| match d { + Declaration::Method(m) => Some(m), + _ => None, + }) + .collect(); + let n: usize = methods + .iter() + .map(|m| { + m.writes.len() + + m.reads.len() + + m.raises.len() + + m.calls.len() + + m.guarded_writes.len() + }) + .sum(); + (c.qualified_name(), n) + }) + .filter(|(_, n)| *n > 0) + .collect(); + body_fact_counts.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); + if !body_fact_counts.is_empty() { + eprintln!("\n[{subtree}] top classes by body-fact count:"); + for (name, n) in body_fact_counts.iter().take(10) { + eprintln!(" {name}: {n} body facts"); + } + } + // Emit the full ndjson manifest — what lance-graph's SPO store + the // ladybug-rs codegen consume. let mut graph = ModelGraph::new(NAMESPACE); From 838cd6926b2f20db7d42e425b522ca162a310e34 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:34 +0000 Subject: [PATCH 06/19] Re-date the census claims now that the harvest example reports it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both docs said the corpus census was a one-off that nothing re-runs. The harvest example now prints it, so that is no longer the limit. The limit that remains is narrower and worth stating precisely: the example prints and does not assert, so a regression would appear in the output and fail nothing. Records the header-only numbers and the cross-check that makes them worth trusting — the 381 methods carrying a body fact agree exactly with an independent count of body-predicate subjects in the emitted ndjson. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/knowledge/fuzzy-recipe-codebook.md | 2 +- .claude/plans/cpp-spo-probes-v1.md | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.claude/knowledge/fuzzy-recipe-codebook.md b/.claude/knowledge/fuzzy-recipe-codebook.md index 10f651a5e2fb02..149c09db6437e4 100644 --- a/.claude/knowledge/fuzzy-recipe-codebook.md +++ b/.claude/knowledge/fuzzy-recipe-codebook.md @@ -84,7 +84,7 @@ landed.) > | `ruff_ruby_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | reference — cook here first | > | `ruff_csharp_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | syntax-only (SemanticModel upgrade pending); helpers via `has_visibility`; tested end-to-end on a real production C# corpus (~97k triples) | > | `ruff_python_spo` | ~ | ✅ | ✅ | ~ | ✗ | reads/raises only; **needs writes/calls/helpers** | -> | `ruff_cpp_spo` | ✅ | ✅ | ✅ | ✅ | ✗ | **(2026-09-07)** wired end-to-end (commit `f068a3f`): harvested from real bodies via libclang 18, same predicates/objects/tiers as `Function`, classified via the now-generic `recipe::classify`; unit- and hermetic-fixture-proven (16 guards, disable-verified) — a one-off corpus census exists (ladybug `src/catalog`, 2794 methods, 20.7% reaching a non-`Empty` centroid; see `.claude/plans/cpp-spo-probes-v1.md`) but **nothing re-runs it**, so C++ has a measurement and not yet a gate; J1 hit-rate is still open, because that corpus contains no lazy-init writes to hit (helpers column unchanged: this commit didn't touch it, and C++ doesn't split private methods into a separate pool the way Ruby does — see the dated note below) | +> | `ruff_cpp_spo` | ✅ | ✅ | ✅ | ✅ | ✗ | **(2026-09-07)** wired end-to-end (commit `f068a3f`): harvested from real bodies via libclang 18, same predicates/objects/tiers as `Function`, classified via the now-generic `recipe::classify`; unit- and hermetic-fixture-proven (16 guards, disable-verified) — a corpus census is now re-runnable (`examples/harvest_ladybug.rs` reports the five fact totals plus a full `RecipeCentroid` census; see `.claude/plans/cpp-spo-probes-v1.md` for the numbers) but **nothing asserts it**, so C++ has a repeatable measurement and still not a gate — a regression would print quietly and fail nothing; J1 hit-rate is still open, because that corpus contains no lazy-init writes to hit (helpers column unchanged: this commit didn't touch it, and C++ doesn't split private methods into a separate pool the way Ruby does — see the dated note below) | > > The fingerprint predicates are already in the shared IR > (`ruff_spo_triplet::Function`) and `expand()` already emits them — a frontend diff --git a/.claude/plans/cpp-spo-probes-v1.md b/.claude/plans/cpp-spo-probes-v1.md index c1913ee244b5da..816baefb9421ae 100644 --- a/.claude/plans/cpp-spo-probes-v1.md +++ b/.claude/plans/cpp-spo-probes-v1.md @@ -664,13 +664,17 @@ codebook classifies on (`writes`/`reads`/`raises`/`calls` + the J1 - **Correction, same day** — an earlier draft of this entry said the feature had "zero corpus-scale numbers". A one-off census HAS since been run, and - its numbers are below; what remains missing is a *committed, re-runnable* - census, because `crates/ruff_cpp_spo/examples/harvest_ladybug.rs` reports - only the signature plane (`inherits_from` trees, `has_function` top-25, - pure-virtual counts, `virtually_overrides`, field/template/static-assert - counts) and never calls `recipe::classify`. So the numbers below are a - measurement, not a gate: nothing re-runs them, and nothing fails if they - regress. + its numbers are below. **Second correction, same day:** + `crates/ruff_cpp_spo/examples/harvest_ladybug.rs` now reports the body arm + too — the five fact totals, a full `RecipeCentroid` census, and a + top-classes-by-body-fact list — so the census is re-runnable rather than + one-off. What is still missing is a *gate*: the example prints, it does not + assert, so a regression would show up in the output and fail nothing. On + `src/include/catalog` (headers only) it reports 2424 methods, 381 with at + least one body fact, census `Observe=283 Compute=62 Normalize=19 + Cascade=10 Guard=7` — and that 381 agrees exactly with an independent count + of body-predicate subjects in the emitted ndjson, which is the cross-check + that makes the census trustworthy rather than merely present. `extract_tree` over `ladybug/src/catalog` (358 classes), every `CppMethod` through `recipe::classify`: From c1f548c935736c84cc13f464296998d049cfe1fd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:34 +0000 Subject: [PATCH 07/19] Correct the stale Python row in the recipe-codebook coverage table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking the Python row while correcting the C++ one turned up the same defect. The table said `ruff_python_spo` had "reads/raises only" and needed writes, calls and helpers. It has had all of them for some time: `functions.rs`'s `BodyWalker` populates every fact set, including the J1 guard across three spellings and `calls` over a closed mutator set with a receiver label, and `dto_arm_writes_guarded_writes_and_calls` asserts them end-to-end with a negative case for a local assignment. The crate's own `lib.rs` quotes this very row as the thing it was closing, so the table has been contradicted from inside the repo and nothing noticed. Also names what the `helpers` column means, because it did not mean the same thing in every row. For Ruby it is a real capability — private defs live in a separate pool and would otherwise be dropped, which is the hook loss this document opens with. Python has no visibility keyword and C++ keeps every method in one collection with its own access field, so neither has a subset to split off. Those two rows now read n/a rather than a cross, which was inviting the reading that a frontend was losing hooks when it was not. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/knowledge/fuzzy-recipe-codebook.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.claude/knowledge/fuzzy-recipe-codebook.md b/.claude/knowledge/fuzzy-recipe-codebook.md index 149c09db6437e4..3403527bb2647d 100644 --- a/.claude/knowledge/fuzzy-recipe-codebook.md +++ b/.claude/knowledge/fuzzy-recipe-codebook.md @@ -74,17 +74,29 @@ landed.) > **This is the "DTO-arm shape" C++ / Python still need.** As of 2026-07-06 > `ruff_ruby_spo` and `ruff_csharp_spo` emit the full quartet + helpers. -> **[2026-09-07 correction: no longer true for C++ — the arm is now wired -> end-to-end, see the row below and the dated note that follows the table. -> Python is unaffected by this update and still needs it.]** +> **[2026-09-07 correction: no longer true for EITHER. C++'s arm was wired +> end-to-end that day (see the row below). Python's was already complete and +> the table had simply not been updated — checking it while correcting the +> C++ row is what turned that up. The sentence above is retained as the +> historical claim it was, not as current state.]** > Coverage: > > | frontend | writes | reads | raises | calls | helpers | verdict | > | ----------------- | :----: | :---: | :----: | :---: | :-----: | ----------------------------------------------------------------------------------------------------------------------------- | > | `ruff_ruby_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | reference — cook here first | > | `ruff_csharp_spo` | ✅ | ✅ | ✅ | ✅ | ✅ | syntax-only (SemanticModel upgrade pending); helpers via `has_visibility`; tested end-to-end on a real production C# corpus (~97k triples) | -> | `ruff_python_spo` | ~ | ✅ | ✅ | ~ | ✗ | reads/raises only; **needs writes/calls/helpers** | -> | `ruff_cpp_spo` | ✅ | ✅ | ✅ | ✅ | ✗ | **(2026-09-07)** wired end-to-end (commit `f068a3f`): harvested from real bodies via libclang 18, same predicates/objects/tiers as `Function`, classified via the now-generic `recipe::classify`; unit- and hermetic-fixture-proven (16 guards, disable-verified) — a corpus census is now re-runnable (`examples/harvest_ladybug.rs` reports the five fact totals plus a full `RecipeCentroid` census; see `.claude/plans/cpp-spo-probes-v1.md` for the numbers) but **nothing asserts it**, so C++ has a repeatable measurement and still not a gate — a regression would print quietly and fail nothing; J1 hit-rate is still open, because that corpus contains no lazy-init writes to hit (helpers column unchanged: this commit didn't touch it, and C++ doesn't split private methods into a separate pool the way Ruby does — see the dated note below) | +> | `ruff_python_spo` | ✅ | ✅ | ✅ | ✅ | n/a | **(2026-09-07 correction)** the row read `~ ✅ ✅ ~ ✗ — reads/raises only; needs writes/calls/helpers`, and was stale: `functions.rs`'s `BodyWalker` populates all five sets (writes, the J1 `guarded_writes` across three guard spellings, and `calls` over a closed `ORM_MUTATORS` set with a receiver label), asserted end-to-end by `dto_arm_writes_guarded_writes_and_calls` including the negative case that a local assignment is not a write. The crate's own `lib.rs` quotes this row as the thing it was closing. **`helpers` is n/a here, not missing**: Python has no visibility keyword, so every `Stmt::FunctionDef` in a model body already lands in `Model::functions` and there is no non-routable subset to split off (`functions.rs` module doc). | +> | `ruff_cpp_spo` | ✅ | ✅ | ✅ | ✅ | n/a | **(2026-09-07)** wired end-to-end (commit `f068a3f`): harvested from real bodies via libclang 18, same predicates/objects/tiers as `Function`, classified via the now-generic `recipe::classify`; unit- and hermetic-fixture-proven (16 guards, disable-verified) — a corpus census is now re-runnable (`examples/harvest_ladybug.rs` reports the five fact totals plus a full `RecipeCentroid` census; see `.claude/plans/cpp-spo-probes-v1.md` for the numbers) but **nothing asserts it**, so C++ has a repeatable measurement and still not a gate — a regression would print quietly and fail nothing; J1 hit-rate is still open, because that corpus contains no lazy-init writes to hit (helpers reads `n/a` for the reason the note under the table gives: C++ keeps every method in one collection with its own `access` field, so there is no separate pool to be missing) | +> +> **Reading the `helpers` column:** it does NOT mean the same thing in every +> row. For Ruby it is a real capability — private defs live in a separate +> `Model::helpers` pool and would otherwise be dropped, which is the ~80% +> hook loss this document opens with. For Python and C++ there is nothing to +> split: Python has no visibility keyword at all, and C++ keeps every method +> in one `methods` collection carrying its own `access` field (and has emitted +> `has_visibility` since before the arm landed). Those two rows read `n/a` +> rather than `✗` as of 2026-09-07, because a `✗` there invited exactly the +> misreading that a frontend was losing hooks when it was not. > > The fingerprint predicates are already in the shared IR > (`ruff_spo_triplet::Function`) and `expand()` already emits them — a frontend From a8cb3661b027f8b9ec7e502634f655bfc66e94f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:24:34 +0000 Subject: [PATCH 08/19] Name the frontends that actually fill the shared IR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crate description and module doc both said the Python frontend was `ruff_python_dto_check` and that a Ruby one was future work. Neither holds. `ruff_python_dto_check` does not depend on this crate, never builds a `ModelGraph`, and emits its own JSON bundle schema — it is a separate harvester, not this crate's Python frontend. The real one is `ruff_python_spo`, whose own module doc says so. `ruff_ruby_spo` is not future either; the recipe codebook calls it the reference frontend. Names the four that fill the IR directly (Odoo, Rails, Flask-SQLAlchemy, C++) and explains why the fifth does not: Roslyn is .NET-only, so `ruff_csharp_spo` parses out of process and joins at the ndjson contract instead. That is why it references no `ModelGraph` — previously readable as a gap rather than a design. The old claim is left in the doc as a note rather than deleted, because a reader who followed it once should be able to see it was corrected and why. The description is crate metadata, so it was wrong in the place least likely to be checked and most likely to be published. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_spo_triplet/Cargo.toml | 2 +- crates/ruff_spo_triplet/src/lib.rs | 33 ++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/ruff_spo_triplet/Cargo.toml b/crates/ruff_spo_triplet/Cargo.toml index 60d1299b791828..e9dff8e32539f7 100644 --- a/crates/ruff_spo_triplet/Cargo.toml +++ b/crates/ruff_spo_triplet/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" publish = false edition = "2024" rust-version = "1.86" -description = "Language-agnostic SPO (Subject-Predicate-Object) triplet expansion. Turns a neutral ModelGraph IR — entities, properties, functions, and their compute/guard/traversal edges — into deterministic NARS-weighted RDF-shape triples (ndjson). The Python/Odoo frontend (ruff_python_dto_check) and a future Ruby/Rails frontend (OpenProject) both fill the same IR and call the same expander, so the downstream SPO graph is identical regardless of source language." +description = "Language-agnostic SPO (Subject-Predicate-Object) triplet expansion. Turns a neutral ModelGraph IR — entities, properties, functions, and their compute/guard/traversal edges — into deterministic NARS-weighted RDF-shape triples (ndjson). Four frontends fill the same IR and call the same expander — ruff_python_spo (Odoo), ruff_ruby_spo (OpenProject/Rails), ruff_sqlalchemy_spo (Flask-SQLAlchemy) and ruff_cpp_spo (C++/libclang) — and ruff_csharp_spo joins at the ndjson contract instead, because Roslyn is .NET-only and its parse runs out of process. The downstream SPO graph is identical regardless of source language." license = "MIT" [lib] diff --git a/crates/ruff_spo_triplet/src/lib.rs b/crates/ruff_spo_triplet/src/lib.rs index 408dc4171fb10e..a6553408ca17e3 100644 --- a/crates/ruff_spo_triplet/src/lib.rs +++ b/crates/ruff_spo_triplet/src/lib.rs @@ -8,13 +8,26 @@ //! directly into the `lance_graph` SPO store. //! //! It exists so that **business logic extracted from different source -//! languages produces the same ontology graph**. The Python/Odoo frontend -//! (`ruff_python_dto_check`) and a future Ruby/Rails frontend (`OpenProject`) -//! both: +//! languages produces the same ontology graph**. Four frontends fill the IR +//! directly — `ruff_python_spo` (Odoo), `ruff_ruby_spo` (OpenProject/Rails), +//! `ruff_sqlalchemy_spo` (Flask-SQLAlchemy) and `ruff_cpp_spo` (C++ via +//! libclang) — and each one: //! -//! 1. parse their own AST, -//! 2. fill a [`ModelGraph`] (the only language-specific work), and -//! 3. call [`expand`] + [`ndjson::to_ndjson`]. +//! 1. parses its own AST, +//! 2. fills a [`ModelGraph`] (the only language-specific work), and +//! 3. calls [`expand`] + [`ndjson::to_ndjson`]. +//! +//! `ruff_csharp_spo` joins one step later: Roslyn is .NET-only, so its parse +//! runs out of process and writes [`Triple`] ndjson directly. Its seam is the +//! ndjson contract rather than the IR, which is why it does not name +//! [`ModelGraph`] anywhere. +//! +//! Note for anyone following an older version of this paragraph: it named +//! `ruff_python_dto_check` as the Python frontend, and Ruby as future work. +//! Neither is so. `ruff_python_dto_check` is a separate harvester with its own +//! JSON bundle schema; it does not depend on this crate and never builds a +//! [`ModelGraph`]. `ruff_ruby_spo` is not future — the recipe codebook calls +//! it the reference frontend. //! //! The triple vocabulary, the provenance/truth calibration, and the IRI //! shape live here once. A new language is a new frontend, not a new @@ -22,9 +35,11 @@ //! //! ```text //! Python AST ─┐ -//! ├─► ModelGraph (IR) ─► expand() ─► Vec ─► ndjson ─► SPO store -//! Ruby AST ──┘ ▲ ▲ ▲ -//! language-specific THIS CRATE THIS CRATE +//! Ruby AST ─┤ +//! C++ AST ─┼─► ModelGraph (IR) ─► expand() ─► Vec ─► ndjson ─► SPO store +//! SQLAlchemy ─┘ ▲ ▲ ▲ ▲ +//! language-specific THIS CRATE THIS CRATE │ +//! C# (Roslyn, out of process) ──────────────────────────────────────┘ //! ``` //! //! # The triple schema From 249e1f9ac53d33d0fa455d70825a83e081bb08db Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 20:36:33 +0000 Subject: [PATCH 09/19] Make the ref-qualifier part of C++ method identity Review finding, verified before fixing. A class may declare `value() &` and `value() &&`. They agree on name, parameter types and cv-qualifier, so the method IRI could not tell them apart and both collapsed onto one node under the `(s, p, o)` dedup. A probe confirmed it: two ref-qualified overloads with different writes produced one subject carrying both. The body arm is what made that harmful rather than merely untidy. Before it, colliding overloads shared signature triples; now they would share each other's writes and calls, and `merge_body_arms` would copy whichever definition it reached first onto both. `CppMethod` gains a `ref_qualifier`, populated from libclang, carried in the IRI suffix beside the cv-qualifier, recovered in reassembly by reading that suffix, and matched in the merge key. Reassembly parses it from the IRI rather than a new predicate, because the IRI already carries it unambiguously and minting one would be an ontology change for a fact that is not missing. Three guards, each disable-verified red-then-green: dropping it from the IRI, from the merge key, and from what the walker reads. The last needs a real parse to be meaningful, since a hand-built fixture cannot tell a populated field from one that is always empty. Two more findings from the same review: * the harvest example recursed with `Path::is_dir`, which follows symlinks, so a directory symlink to an ancestor loops forever. The crate's own `collect_cpp_files` already guards this with `DirEntry::file_type`, for the same reason, so the example had reintroduced a solved bug. Fixed the same way. It carries no test, because an example is not a test target; the crate's equivalent is covered by `extract_tree_skips_directory_symlink_cycles`. * the codebook said the fingerprint is four fact sets while the table beneath it listed five. Not cosmetic: `classify` tests `guarded_writes` before `Compute` and `Normalize`, so a frontend trusting the sentence over the table would misclassify every guarded default. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/knowledge/fuzzy-recipe-codebook.md | 9 ++- .../ruff_cpp_spo/examples/harvest_ladybug.rs | 13 ++- crates/ruff_cpp_spo/src/clang_walker.rs | 78 +++++++++++++++++- crates/ruff_cpp_spo/src/lib.rs | 76 +++++++++++++++++- crates/ruff_spo_triplet/src/expand.rs | 80 ++++++++++++++++++- crates/ruff_spo_triplet/src/ir.rs | 37 +++++++++ crates/ruff_spo_triplet/src/lib.rs | 6 +- crates/ruff_spo_triplet/src/reassemble.rs | 47 +++++++++-- 8 files changed, 324 insertions(+), 22 deletions(-) diff --git a/.claude/knowledge/fuzzy-recipe-codebook.md b/.claude/knowledge/fuzzy-recipe-codebook.md index 3403527bb2647d..79ebb3760a7bdd 100644 --- a/.claude/knowledge/fuzzy-recipe-codebook.md +++ b/.claude/knowledge/fuzzy-recipe-codebook.md @@ -52,7 +52,7 @@ on code: ## 2. The fingerprint — the DTO arm (this is what every frontend must emit) -The fingerprint is FOUR fact sets per method, on `ruff_spo_triplet::Function` +The fingerprint is FIVE fact sets per method, on `ruff_spo_triplet::Function` (and, since 2026-09-07, on `ruff_spo_triplet::CppMethod` too — see the dated note beneath the coverage table below for why C++ needed its own copy of these fields rather than reusing `Function`'s): @@ -65,6 +65,13 @@ these fields rather than reusing `Function`'s): | `calls` | `calls` | Inferred | mutator dispatches `"receiver.method"` | | `guarded_writes` | `writes_if_blank` | **Authoritative** | writes guarded by a blank/nil test on the same field — the **J1** fact (§5), splits SelfMap into default vs normalize | +The count above read FOUR until 2026-09-07, while the table under it listed +five rows. That is not a cosmetic slip: `recipe::classify` tests +`guarded_writes` BEFORE `Compute` and `Normalize`, so a frontend that read the +sentence rather than the table and harvested only the quartet would classify +every guarded default as a compute or a normalize. A frontend joins this +codebook by populating five sets, not four. + Plus the **visibility split**: hook targets are conventionally *private*, so a frontend that drops private defs cannot resolve most hooks. `ruff_ruby_spo` carries them in `Model::helpers` (walked identically, kept out of the routable diff --git a/crates/ruff_cpp_spo/examples/harvest_ladybug.rs b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs index ce76f34e60cf8a..b2e60718bc6d72 100644 --- a/crates/ruff_cpp_spo/examples/harvest_ladybug.rs +++ b/crates/ruff_cpp_spo/examples/harvest_ladybug.rs @@ -63,9 +63,18 @@ fn collect_headers(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let entry = entry?; let path = entry.path(); - if path.is_dir() { + // Ask the ENTRY for its type rather than the path: `Path::is_dir` + // follows symlinks, so a directory symlink pointing at an ancestor + // recurses forever and the harvest never finishes. Symlinks are + // skipped entirely — a header tree's translation units are real + // files. This is the same guard, for the same reason, that + // `ruff_cpp_spo::collect_cpp_files` already carries. + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { collect_headers(&path, out)?; - } else if path.extension().is_some_and(|x| x == "h") { + } else if file_type.is_file() && path.extension().is_some_and(|x| x == "h") { out.push(path); } } diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index 7de1dd60d1ef5f..163447ee4b3855 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -40,9 +40,12 @@ use std::fmt; use std::path::Path; use clang::diagnostic::Severity; -use clang::{Accessibility, Clang, Entity, EntityKind, ExceptionSpecification, Index}; +use clang::{ + Accessibility, Clang, Entity, EntityKind, ExceptionSpecification, Index, RefQualifier, +}; use ruff_spo_triplet::{ - CppAccess, CppBase, CppField, CppFriend, CppMethod, CppTemplate, CppTemplateKind, + CppAccess, CppBase, CppField, CppFriend, CppMethod, CppRefQualifier, CppTemplate, + CppTemplateKind, }; use crate::{CppClass, CppEnum, CppFunction, Declaration}; @@ -849,6 +852,16 @@ fn build_method(m: &Entity, arm: Option<&BodyArmConfig>) -> CppMethod { raises: body.raises, calls: body.calls, guarded_writes: body.guarded_writes, + // `void f() &` / `void f() &&`. Part of the method's identity, so it + // has to reach the IRI: two ref-qualified overloads otherwise share + // one node and the body-arm merge copies one's facts onto the other. + ref_qualifier: m + .get_type() + .and_then(|t| t.get_ref_qualifier()) + .map(|q| match q { + RefQualifier::LValue => CppRefQualifier::LValue, + RefQualifier::RValue => CppRefQualifier::RValue, + }), } } @@ -1734,6 +1747,67 @@ void Svc::finish(int v) { status_ = v; repo_.Save(); } assert_eq!(finish.calls, ["repo_.Save"]); } + /// libclang really does report the ref-qualifier, and each overload keeps + /// its own body. + /// + /// The IR-level falsifier for this builds `CppMethod`s by hand, so it + /// cannot tell a populated field from one that is always `None`. This + /// parses the real thing: if `build_method` stopped reading the qualifier, + /// all three overloads below would report `None` and the assertions fail. + #[test] + fn libclang_reports_the_ref_qualifier_and_each_overload_keeps_its_body() { + let src = r#" +struct Holder { + int lvalue_only_; + int rvalue_only_; + int plain_only_; + void value() & { lvalue_only_ = 1; } + void value() && { rvalue_only_ = 2; } + void value() { plain_only_ = 3; } +}; +"#; + let dir = std::env::temp_dir().join("cpp_refqual"); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("f.cpp"); + std::fs::write(&path, src).expect("write fixture"); + + let _guard = CLANG_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (classes, _) = walk_tu_configured( + &path, + &["-std=c++17".to_string()], + Some(&BodyArmConfig::default()), + ) + .expect("walk"); + let holder = classes.iter().find(|c| c.name == "Holder").expect("Holder"); + let methods: Vec<&CppMethod> = holder + .declarations + .iter() + .filter_map(|d| match d { + Declaration::Method(m) if m.name == "value" => Some(m), + _ => None, + }) + .collect(); + assert_eq!(methods.len(), 3, "three overloads"); + + let by_qualifier = |q: Option| { + methods + .iter() + .find(|m| m.ref_qualifier == q) + .unwrap_or_else(|| panic!("no overload with qualifier {q:?}")) + }; + assert_eq!( + by_qualifier(Some(CppRefQualifier::LValue)).writes, + ["lvalue_only_"] + ); + assert_eq!( + by_qualifier(Some(CppRefQualifier::RValue)).writes, + ["rvalue_only_"] + ); + assert_eq!(by_qualifier(None).writes, ["plain_only_"]); + } + /// The opt-out: a signature-only walk parses no bodies, so every arm is /// empty while the signature plane is unchanged. /// diff --git a/crates/ruff_cpp_spo/src/lib.rs b/crates/ruff_cpp_spo/src/lib.rs index 4d5108d7cca4c6..cbcdc708c29c36 100644 --- a/crates/ruff_cpp_spo/src/lib.rs +++ b/crates/ruff_cpp_spo/src/lib.rs @@ -349,10 +349,13 @@ fn walk_files(files: &[std::path::PathBuf], args: &[String]) -> Result = triples + .iter() + .filter(|t| t.p == "writes_field") + .map(|t| t.s.as_str()) + .collect(); + assert_eq!(carriers.len(), 3, "overloads shared a node: {carriers:?}"); + assert!(carriers.contains("cpp:Holder.value() &")); + assert!(carriers.contains("cpp:Holder.value() &&")); + assert!(carriers.contains("cpp:Holder.value()")); + + // Each node carries ONLY its own write — the contamination this + // guards against is a body fact appearing on the wrong overload. + let writes_of = |iri: &str| { + triples + .iter() + .filter(|t| t.s == iri && t.p == "writes_field") + .map(|t| t.o.as_str()) + .collect::>() + }; + assert_eq!( + writes_of("cpp:Holder.value() &"), + ["cpp:Holder.lvalue_only_"] + ); + assert_eq!( + writes_of("cpp:Holder.value() &&"), + ["cpp:Holder.rvalue_only_"] + ); + assert_eq!(writes_of("cpp:Holder.value()"), ["cpp:Holder.plain_only_"]); + } + /// The C++ body arm expands to the SAME five predicates, with the SAME /// object encoding, as a Ruby/Python `Function` body: own members join /// the `cpp_field` node, exceptions take the `exc:` namespace, calls stay diff --git a/crates/ruff_spo_triplet/src/ir.rs b/crates/ruff_spo_triplet/src/ir.rs index 473f73ea7a6f53..5e8182e94893d0 100644 --- a/crates/ruff_spo_triplet/src/ir.rs +++ b/crates/ruff_spo_triplet/src/ir.rs @@ -792,6 +792,41 @@ pub struct CppMethod { /// (Authoritative). Sorted, deduped. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub guarded_writes: Vec, + /// `void f() &` / `void f() &&` — the ref-qualifier, part of the method's + /// identity rather than a property. `None` for the unqualified case. It + /// rides the method IRI's suffix alongside the cv-qualifier; see + /// [`CppRefQualifier`] for why leaving it out silently merges two + /// different functions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ref_qualifier: Option, +} + +/// The ref-qualifier of a member function (`void f() &` / `void f() &&`). +/// +/// Part of a method's IDENTITY, not merely a property: a class may declare +/// both `f() &` and `f() &&` with the same name, parameter types and +/// cv-qualifier, and they are different functions with different bodies. +/// Without this in the method IRI the two collapse onto one node under the +/// `(s, p, o)` dedup, and a body-arm merge then copies one overload's facts +/// onto the other. `None` is the unqualified case, which is almost every +/// method. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum CppRefQualifier { + /// `void f() &` — callable on an lvalue. + LValue, + /// `void f() &&` — callable on an rvalue. + RValue, +} + +impl CppRefQualifier { + /// The C++ spelling, as it appears in a method IRI's suffix. + #[must_use] + pub fn spelling(self) -> &'static str { + match self { + Self::LValue => "&", + Self::RValue => "&&", + } + } } /// `constexpr` vs `consteval` compile-time markers. @@ -937,6 +972,7 @@ mod dto_surface_tests { "param_types", "raises", "reads", + "ref_qualifier", "requires_clause", "return_type", "writes", @@ -1010,6 +1046,7 @@ mod dto_surface_tests { raises: vec!["E".into()], calls: vec!["repo_.Save".into()], guarded_writes: vec!["w_".into()], + ref_qualifier: Some(CppRefQualifier::LValue), } } diff --git a/crates/ruff_spo_triplet/src/lib.rs b/crates/ruff_spo_triplet/src/lib.rs index a6553408ca17e3..7ec16eead6838e 100644 --- a/crates/ruff_spo_triplet/src/lib.rs +++ b/crates/ruff_spo_triplet/src/lib.rs @@ -95,9 +95,9 @@ pub use expand::expand; pub use ir::{ ActsAs, AssocDecl, AssocKind, AttrDecl, AttrKind, Callback, ConcernKind, ConcernRef, ConstexprKind, CppAccess, CppBase, CppField, CppFriend, CppMacroUse, CppMethod, - CppStaticAssert, CppTemplate, CppTemplateKind, Delegation, DslCall, DynMethod, Field, Function, - GemDsl, GemKind, Model, ModelGraph, ScopeDecl, ScopeKind, StiInfo, UsingRef, Validation, - ValidationKind, + CppRefQualifier, CppStaticAssert, CppTemplate, CppTemplateKind, Delegation, DslCall, DynMethod, + Field, Function, GemDsl, GemKind, Model, ModelGraph, ScopeDecl, ScopeKind, StiInfo, UsingRef, + Validation, ValidationKind, }; pub use nav_digest::build_nav_digest; pub use ndjson::{ParseError, from_ndjson, to_ndjson}; diff --git a/crates/ruff_spo_triplet/src/reassemble.rs b/crates/ruff_spo_triplet/src/reassemble.rs index 6e6afaf5a9aba0..96d35ebf469a2f 100644 --- a/crates/ruff_spo_triplet/src/reassemble.rs +++ b/crates/ruff_spo_triplet/src/reassemble.rs @@ -48,7 +48,7 @@ use std::collections::BTreeMap; use crate::ir::{ ConstexprKind, CppAccess, CppBase, CppField, CppFriend, CppMacroUse, CppMethod, - CppStaticAssert, CppTemplate, CppTemplateKind, Model, ModelGraph, + CppRefQualifier, CppStaticAssert, CppTemplate, CppTemplateKind, Model, ModelGraph, }; use crate::triple::Triple; @@ -302,14 +302,30 @@ pub fn reassemble(triples: &[Triple]) -> ModelGraph { continue; }; let param_types: Vec = acc.params.into_values().collect(); + // Read the ref-qualifier off the IRI's tail rather than from a + // triple: it is part of the identity `expand` encodes there, and + // minting a predicate for it would be an ontology change for a fact + // the IRI already carries unambiguously. That is also why it is not a + // `MethodAcc` field — nothing routes into it during the triple pass. Everything after the + // closing paren is `expand`'s own suffix, so ` &&` / ` &` there can + // only be the qualifier: a parameter type ending in `&` sits INSIDE + // the parens and leaves the IRI ending in `)`. + let ref_qualifier = if method_iri.ends_with(" &&") { + Some(CppRefQualifier::RValue) + } else if method_iri.ends_with(" &") { + Some(CppRefQualifier::LValue) + } else { + None + }; // Reconstruct the exact suffix `expand` built — including the ` const` // cv-qualifier when the method is const — so the prefix/suffix strip // recovers the bare name. `is_const` was collected from the property // triple in pass 3, so it is available here at finalize. let suffix = format!( - "({}){}", + "({}){}{}", param_types.join(","), - if acc.is_const { " const" } else { "" } + if acc.is_const { " const" } else { "" }, + ref_qualifier.map_or_else(String::new, |q| format!(" {}", q.spelling())) ); let class_prefix = format!("{class_iri}."); let name = method_iri @@ -335,6 +351,7 @@ pub fn reassemble(triples: &[Triple]) -> ModelGraph { raises: acc.raises, calls: acc.calls, guarded_writes: acc.guarded_writes, + ref_qualifier, }); } @@ -411,12 +428,22 @@ fn canonicalize_cpp(graph: &mut ModelGraph) { arm.dedup(); } } + // The sort key mirrors the method IRI's identity exactly — name, + // params, cv-qualifier, ref-qualifier — so an overload set orders + // deterministically on both the reassembled and the projected side. model.methods.sort_by(|a, b| { - (a.name.as_str(), &a.param_types, a.is_const).cmp(&( - b.name.as_str(), - &b.param_types, - b.is_const, - )) + ( + a.name.as_str(), + &a.param_types, + a.is_const, + a.ref_qualifier.map(CppRefQualifier::spelling), + ) + .cmp(&( + b.name.as_str(), + &b.param_types, + b.is_const, + b.ref_qualifier.map(CppRefQualifier::spelling), + )) }); model.methods.dedup(); model.templates.sort_by(|a, b| { @@ -499,6 +526,7 @@ mod tests { raises: vec!["BadStatus".to_string()], calls: vec!["repo_.Save".to_string()], guarded_writes: vec!["recognizer_".to_string()], + ref_qualifier: None, }); rec.methods.push(CppMethod { name: "Clear".to_string(), @@ -518,6 +546,7 @@ mod tests { raises: Vec::new(), calls: Vec::new(), guarded_writes: Vec::new(), + ref_qualifier: None, }); rec.methods.push(CppMethod { name: "kMaxRating".to_string(), @@ -537,6 +566,7 @@ mod tests { raises: Vec::new(), calls: Vec::new(), guarded_writes: Vec::new(), + ref_qualifier: None, }); rec.methods.push(CppMethod { name: "operator==".to_string(), @@ -556,6 +586,7 @@ mod tests { raises: Vec::new(), calls: Vec::new(), guarded_writes: Vec::new(), + ref_qualifier: None, }); rec.templates.push(CppTemplate { kind: CppTemplateKind::Specialisation, From fae369ab94369f77ecbb2838e5bd045db63a29db Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 22:07:53 +0000 Subject: [PATCH 10/19] Preserve the C++ body's sequence, duplicates and scope structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The body arm ends by collapsing its traversal into sets: `method_body_arm` runs `facts.sort(); facts.dedup();` over all five vectors, unconditionally. And `walk_body` gives `ForStmt` / `WhileStmt` / `DoStmt` / range-`for` / `SwitchStmt` / `TryStmt` / `LambdaExpr` no match arm at all — their contents are visited, their boundaries recorded nowhere. So a write inside a loop, two writes in a row, and a call in an else-branch are all unrepresentable, and the recurring shapes a behavioral vocabulary would be made of are exactly the ones that need order and scope. That is right for `recipe::classify`, which is a set-membership classifier by design. It is wrong as the only thing the walker emits. `events` is a second walk over the same cursor that preserves three things separately: identity in a deduped symbol table, order in an event stream that is never sorted, structure in a scope tree. The same fact occurring twice is two events pointing at one symbol. Additive: `method_body_arm`, `BodyArm`, `CppMethod`, `expand` and the emitted ndjson are unchanged, and `harvest_ladybug` over `catalog` reproduces its 2,106,907 bytes byte-identically. `events_collapse_to_the_shipped_five_sets` asserts the two walkers agree on what a fact IS, so the difference is only what is preserved about it. The shipped arm runs on the same cursor in the same parse and its verdict rides in `methods.tsv`, so a consumer can compare without a second walk making the difference ambiguous. The alphabet is 20 primitive kinds. There is deliberately no RAII, iterator- loop, virtual-dispatch or move kind: those are candidate learned tokens, and emitting them would write a downstream tokenizer's answer into its input. Identities are verbatim, never hashed. libclang exposes no CFG, so `control` carries only structurally-certain relations and every event keeps its source byte anchor — lexical order stays checkable against traversal order instead of being silently conflated with it. Three defects the control fixture caught, whose expected sequence was written before the code existed: - a call's own callee is a `MemberRefExpr`, so walking it emitted a read of a member that does not exist — the exact trap the shipped walker documents; - braced arms opened a redundant `Block`, which would have taught a tokenizer brace style rather than behavior; - `do { } while (c)` puts its body FIRST, so the "last child is the body" rule walked the condition as a statement and wrapped the real body in a Block. Where libclang disagreed with the hand-written expectation — the `for` header emits its condition marker after init/cond/inc, which is not source order — the disagreement is recorded rather than reconciled. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/plans/behavioral-ore-v1.md | 96 ++ crates/ruff_cpp_spo/Cargo.toml | 4 + .../ruff_cpp_spo/examples/harvest_events.rs | 331 ++++ crates/ruff_cpp_spo/src/clang_walker.rs | 16 +- crates/ruff_cpp_spo/src/events.rs | 1500 +++++++++++++++++ crates/ruff_cpp_spo/src/lib.rs | 6 + 6 files changed, 1945 insertions(+), 8 deletions(-) create mode 100644 .claude/plans/behavioral-ore-v1.md create mode 100644 crates/ruff_cpp_spo/examples/harvest_events.rs create mode 100644 crates/ruff_cpp_spo/src/events.rs diff --git a/.claude/plans/behavioral-ore-v1.md b/.claude/plans/behavioral-ore-v1.md new file mode 100644 index 00000000000000..c573456e81f609 --- /dev/null +++ b/.claude/plans/behavioral-ore-v1.md @@ -0,0 +1,96 @@ +# Ordered behavioral ore for C++ — plan v1 + +**Status:** IN PROGRESS. The extractor is the deliverable here; the tokenizer +that consumes it is a separate, downstream experiment and nothing in this plan +asserts a result for it. + +## 1. The baseline defect (this is a FINDING, not plumbing) + +> **The C++ harvest representation cannot carry behavior, because it is a set.** + +Two independent causes, both in `ruff_cpp_spo`: + +1. **`method_body_arm` ends by collapsing the traversal into sets.** + `clang_walker.rs:1011-1025` runs `facts.sort(); facts.dedup();` over all + five vectors, unconditionally, in the one function that produces a + `BodyArm`. Adjacency, repetition and interleaving are gone before any + consumer sees a fact. `expand.rs:904-943` then emits by fixed category + (reads → raises → writes → guarded → calls), alphabetical within category, + so nothing downstream can recover source order either. + +2. **`walk_body` gives the structured constructs no match arm.** + Only `ThrowExpr`, `BinaryOperator`, `CompoundAssignOperator`, + `UnaryOperator`, `CallExpr`, `IfStmt` and `MemberRefExpr` are matched + (`clang_walker.rs:1035-1110`). `ForStmt`, `WhileStmt`, `DoStmt`, + range-for, `SwitchStmt`, `CXXTryStmt`, `CXXCatchStmt`, `LambdaExpr`, + `ReturnStmt` fall through `_ => walk_body(...)`: their CONTENTS are + visited, their BOUNDARIES are not recorded anywhere. A write inside a loop + and a write in straight-line code are the same fact. `IfStmt` is matched + only to thread the J1 guard; it emits no event of its own. + +**Consequence.** The recurring shapes a behavioral vocabulary would be made of +— an iterator loop, a lock/mutate/unlock triple, an RAII scope, a +dispatch-then-propagate chain, a try/catch compensation — are precisely the +ones that require order and scope, and every one of them is unrepresentable +today. A sequence learner run against this output is learning words from a bag +of letters. + +**This is not an argument against the five sets.** They are the right input to +`recipe::classify`, which is a set-membership classifier by design and works. +The defect is that the harvest DISCARDS the sequence rather than emitting both. + +### Measured on the control fixture +See `§4`. The measurement is reported by `ore.py`'s loss table and is repeated +in the commit that lands the extractor; it is not asserted here in advance. + +## 2. What this adds — three things, preserved SEPARATELY + +| preserved | artefact | deduped? | +|---|---|---| +| event identity | `symbols.tsv` | yes — the only place | +| event order | `events.tsv` | **never** | +| structural context | `scopes.tsv` | n/a | + +The same fact occurring twice is two rows in `events.tsv` pointing at one row +in `symbols.tsv`. Keeping the canonical symbol table and the occurrence stream +apart is what lets a consumer choose its own granularity without the extractor +having chosen for it. + +## 3. Neutrality rules — the ore must not pre-solve the experiment + +1. **No inferred motifs in the alphabet.** The 20 kinds are primitive + (`ScopeEnter`/`Read`/`Call`/`Condition`/…). There is no `RAII`, + `IteratorLoop`, `VirtualDispatch` or `Move` kind. A `lock_guard` local is + `Decl` + a type symbol; `std::move(x)` is `Call` + a callee symbol. Those + concepts are candidate LEARNED tokens; emitting them would be the + tokenizer's answer written into its input. +2. **No consumer vocabulary.** Nothing about classids, facets, rails, masks or + any 96-bit layout appears in the ore or in this crate. The ore does not + know those exist. +3. **Identities are verbatim, never hashed.** Different consumers hash + differently; a hash in the ore forces one of them. +4. **Lexical order and control flow stay distinguishable.** Every event carries + its source byte `anchor`. `control` carries only structurally-certain + relations. **libclang's C API exposes no CFG** — successor/predecessor + edges are not available and are never fabricated. A later probe may ask + whether CFG adjacency beats lexical adjacency; it cannot ask it from these + files, and must say so rather than assume the two are the same. + +## 4. The control + +`for (…) { if (…) foo(); else bar(); }` — the expected event sequence was +hand-written BEFORE the extractor existed. Where libclang disagrees (AST child +order for `ForStmt` is init/cond/inc/body, which is not source order), the +disagreement is reported and the `anchor` column keeps both orders separable. +It is not resolved by editing the expectation to match the code. + +## 5. Fences + +- **Additive only.** `method_body_arm`, `BodyArm`, `CppMethod`, `expand` and + the emitted ndjson are byte-identical after this change. A test asserts that + collapsing the new events to the five sets reproduces `method_body_arm` + exactly — proving the change is about what is PRESERVED, not about what a + fact is. +- **No new predicate.** The closed vocabulary is untouched; the ore is TSV + beside the manifest, not a triple. +- The extractor is a second walk over the same cursor, in its own module. diff --git a/crates/ruff_cpp_spo/Cargo.toml b/crates/ruff_cpp_spo/Cargo.toml index 32ef2b336ffac9..d68c37f77716d7 100644 --- a/crates/ruff_cpp_spo/Cargo.toml +++ b/crates/ruff_cpp_spo/Cargo.toml @@ -65,3 +65,7 @@ ruff_cpp_codegen = { workspace = true } [lints] workspace = true + +[[example]] +name = "harvest_events" +required-features = ["libclang"] diff --git a/crates/ruff_cpp_spo/examples/harvest_events.rs b/crates/ruff_cpp_spo/examples/harvest_events.rs new file mode 100644 index 00000000000000..fa6df61f09b1a2 --- /dev/null +++ b/crates/ruff_cpp_spo/examples/harvest_events.rs @@ -0,0 +1,331 @@ +//! Emit the ordered behavioral ore for a C++ corpus — the four TSVs described +//! in `.claude/plans/behavioral-ore-v1.md`. +//! +//! ONE extractor, many consumers: a frequency tokenizer, a behavioral +//! tokenizer and any downstream representation all read these same files. No +//! consumer re-extracts, so extraction differences can never masquerade as +//! representation differences. +//! +//! Env: +//! `ORE_OUT` output directory (default `/tmp/ore`) +//! `ORE_CC_JSON` a `compile_commands.json`; each entry's own `-I`/`-D`/ +//! `-std`/`-isystem` flags are used, so the parse sees what +//! the compiler sees instead of a guessed argument set +//! `ORE_TU_FILTER` substring filter on the TU path +//! `ORE_TU_LIMIT` max TUs +//! `ORE_FILE` a single source file (alternative to `ORE_CC_JSON`) +//! `ORE_ARGS_FILE` one clang argument per line, for `ORE_FILE` +//! +//! ```sh +//! ORE_CC_JSON=build/compile_commands.json ORE_TU_LIMIT=50 ORE_OUT=/tmp/ore \ +//! LIBCLANG_PATH=/usr/lib/llvm-18/lib \ +//! cargo run -p ruff_cpp_spo --features libclang --example harvest_events +//! ``` + +#![expect( + clippy::print_stderr, + reason = "manifest-emission CLI example (mirrors harvest_ladybug)" +)] + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::PathBuf; + +use ruff_cpp_spo::events::{MethodOre, Symbols, walk_tu_events}; + +/// A TSV cell can carry no tab and no newline. C++ identifiers and type +/// spellings contain neither, but a malformed one would silently shift every +/// later column, so the guard is unconditional. +fn cell(s: &str) -> String { + s.replace(['\t', '\n', '\r'], " ") +} + +/// Pull the compile flags out of one `compile_commands.json` entry. Only the +/// flags that change what the parser SEES are kept — `-o`, `-c` and the input +/// file itself would make libclang re-drive a compilation. +fn args_of(cmd: &str) -> Vec { + let toks: Vec = shell_split(cmd); + let mut out = Vec::new(); + let mut i = 0; + while i < toks.len() { + let t = &toks[i]; + if t == "-isystem" || t == "-include" { + if let Some(v) = toks.get(i + 1) { + out.push(t.clone()); + out.push(v.clone()); + } + i += 2; + continue; + } + if t.starts_with("-I") || t.starts_with("-D") || t.starts_with("-std") { + out.push(t.clone()); + } + i += 1; + } + out +} + +/// Minimal POSIX-ish splitter: enough for the quoting cmake emits. +fn shell_split(s: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quote = None::; + let mut esc = false; + for c in s.chars() { + if esc { + cur.push(c); + esc = false; + } else if c == '\\' { + esc = true; + } else if let Some(q) = quote { + if c == q { + quote = None; + } else { + cur.push(c); + } + } else if c == '\'' || c == '"' { + quote = Some(c); + } else if c.is_whitespace() { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } else { + cur.push(c); + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +/// `compile_commands.json` without a JSON dependency: the file is a flat array +/// of objects with string values, and only two keys are needed. +fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { + let mut out = Vec::new(); + let (mut file, mut cmd) = (None::, None::); + for raw in text.lines() { + let line = raw.trim(); + if let Some(v) = json_string_field(line, "\"file\":") { + file = Some(v); + } else if let Some(v) = json_string_field(line, "\"command\":") { + cmd = Some(v); + } + if let (Some(f), Some(c)) = (&file, &cmd) { + out.push((PathBuf::from(f), args_of(c))); + file = None; + cmd = None; + } + } + out +} + +fn json_string_field(line: &str, key: &str) -> Option { + let rest = line.strip_prefix(key)?.trim_start(); + let body = rest.strip_prefix('"')?; + let mut out = String::new(); + let mut esc = false; + for c in body.chars() { + if esc { + out.push(match c { + 'n' => '\n', + 't' => '\t', + other => other, + }); + esc = false; + } else if c == '\\' { + esc = true; + } else if c == '"' { + return Some(out); + } else { + out.push(c); + } + } + None +} + +fn main() -> Result<(), Box> { + let out_dir = PathBuf::from(std::env::var("ORE_OUT").unwrap_or_else(|_| "/tmp/ore".into())); + std::fs::create_dir_all(&out_dir)?; + + let mut units: Vec<(PathBuf, Vec)> = Vec::new(); + if let Ok(cc) = std::env::var("ORE_CC_JSON") { + let text = std::fs::read_to_string(&cc)?; + units = parse_cc_json(&text); + if let Ok(f) = std::env::var("ORE_TU_FILTER") { + units.retain(|(p, _)| p.display().to_string().contains(&f)); + } + if let Ok(n) = std::env::var("ORE_TU_LIMIT") { + let n: usize = n.parse().unwrap_or(usize::MAX); + units.truncate(n); + } + } else if let Ok(f) = std::env::var("ORE_FILE") { + let args = std::env::var("ORE_ARGS_FILE").map_or_else( + |_| Vec::new(), + |p| { + std::fs::read_to_string(p) + .map_or_else(|_| Vec::new(), |t| t.lines().map(str::to_string).collect()) + }, + ); + units.push((PathBuf::from(f), args)); + } else { + return Err("set ORE_CC_JSON or ORE_FILE — this example never invents a corpus".into()); + } + if units.is_empty() { + return Err("no translation units selected".into()); + } + + let mut syms = Symbols::default(); + let mut methods: Vec = Vec::new(); + let (mut bad, mut failed) = (0usize, 0usize); + for (i, (path, args)) in units.iter().enumerate() { + match walk_tu_events(path, args, &mut syms) { + Ok(tu) => { + if !tu.diagnostics.is_empty() { + bad += 1; + if bad <= 5 { + eprintln!( + "[ore] {} error diagnostic(s) in {} — partial AST, facts may be missing\n {}", + tu.diagnostics.len(), + path.display(), + tu.diagnostics[0] + ); + } + } + methods.extend(tu.methods); + } + Err(e) => { + failed += 1; + if failed <= 5 { + eprintln!("[ore] FAILED {}: {e}", path.display()); + } + } + } + if (i + 1) % 25 == 0 { + eprintln!( + "[ore] {}/{} TUs, {} methods, {} symbols", + i + 1, + units.len(), + methods.len(), + syms.len() + ); + } + } + + // A method defined in a header reachable from several TUs is harvested + // once per TU. Keep the FIRST sighting: dropping duplicates here is a + // corpus decision, not a dedup of occurrences — the event stream inside + // each kept method is untouched. + let mut seen = BTreeMap::new(); + for m in methods { + seen.entry(m.iri.clone()).or_insert(m); + } + let methods: Vec = seen.into_values().collect(); + + let (mut ev, mut sc, mut me) = (String::new(), String::new(), String::new()); + let mut hist: BTreeMap<&'static str, usize> = BTreeMap::new(); + let mut n_events = 0usize; + for m in &methods { + for e in &m.events { + *hist.entry(e.kind.as_str()).or_default() += 1; + n_events += 1; + writeln!( + ev, + "{}\t{}\t{}\t{}\t{}\tc{}\t{}\t{}\t{}\t{}\t{}", + cell(&m.iri), + e.seq, + e.kind, + e.subj.as_deref().unwrap_or("-"), + e.obj.as_deref().unwrap_or("-"), + e.scope, + e.parent + .map_or_else(|| "-".to_string(), |p| format!("c{p}")), + e.anchor, + e.control.as_deref().unwrap_or("-"), + e.type_rel.as_deref().unwrap_or("-"), + e.prov.as_str() + )?; + } + for s in &m.scopes { + writeln!( + sc, + "{}\tc{}\t{}\t{}\t{}\t{}\t{}", + cell(&m.iri), + s.id, + s.parent + .map_or_else(|| "-".to_string(), |p| format!("c{p}")), + s.kind.as_str(), + s.depth, + s.enter_seq, + s.exit_seq + )?; + } + writeln!( + me, + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + cell(&m.iri), + cell(&m.tu), + cell(&m.class), + m.events.len(), + m.scopes.len(), + m.centroid, + m.set_reads, + m.set_writes, + m.set_raises, + m.set_calls, + m.set_guarded, + u8::from(m.is_const), + u8::from(m.is_static), + u8::from(m.is_virtual), + u8::from(m.overrides), + m.mkind, + m.access, + m.n_params + )?; + } + let mut sy = String::new(); + for (id, kind, role, name, prov) in syms.rows() { + writeln!( + sy, + "{id}\t{}\t{}\t{}\t{}", + kind.as_str(), + role.as_str(), + cell(name), + prov.as_str() + )?; + } + + std::fs::write(out_dir.join("events.tsv"), &ev)?; + std::fs::write(out_dir.join("scopes.tsv"), &sc)?; + std::fs::write(out_dir.join("methods.tsv"), &me)?; + std::fs::write(out_dir.join("symbols.tsv"), &sy)?; + + eprintln!( + "\n[ore] {} TUs ({} with error diagnostics, {} failed to parse)", + units.len(), + bad, + failed + ); + eprintln!( + "[ore] {} methods, {n_events} events, {} symbols -> {}", + methods.len(), + syms.len(), + out_dir.display() + ); + let with_events = methods.iter().filter(|m| m.events.len() > 2).count(); + let a0_facts: u32 = methods + .iter() + .map(|m| m.set_reads + m.set_writes + m.set_raises + m.set_calls) + .sum(); + eprintln!( + "[ore] methods with a body beyond the empty function scope: {with_events}; \ + A0 five-set facts over the same corpus: {a0_facts}" + ); + eprintln!("[ore] event-kind histogram:"); + let mut h: Vec<_> = hist.into_iter().collect(); + h.sort_by_key(|a| std::cmp::Reverse(a.1)); + for (k, n) in h { + eprintln!(" {k:<11} {n:>8}"); + } + Ok(()) +} diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index 163447ee4b3855..b09d042e1ea781 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -735,7 +735,7 @@ fn enclosing_scopes(e: &Entity) -> Vec { } /// The fully-qualified name of a class-like cursor (`Namespace::Outer::Name`). -fn qualified_name(e: &Entity) -> String { +pub(crate) fn qualified_name(e: &Entity) -> String { let mut parts = enclosing_scopes(e); if let Some(n) = e.get_name() { parts.push(n); @@ -1203,7 +1203,7 @@ fn record_write(arm: &mut BodyArm, lhs: &Entity, guard: Option<&str>) { /// (`arr_[i] = v` and `*ptr_ = v` both write the member). `None` when the /// target is anything else — a local, a parameter, or another object's member. #[cfg(feature = "libclang")] -fn assignment_target(lhs: &Entity) -> Option { +pub(crate) fn assignment_target(lhs: &Entity) -> Option { let mut cur = *lhs; // Bounded: each step descends one AST level, and the depth of an lvalue // expression is finite. The cap only guards against a cyclic cursor graph. @@ -1228,7 +1228,7 @@ fn assignment_target(lhs: &Entity) -> Option { /// [`EntityKind::ThisExpr`]; any other base (`p.x_`, `repo_.Save`) means the /// reference belongs to something else. #[cfg(feature = "libclang")] -fn own_member_name(e: &Entity) -> Option { +pub(crate) fn own_member_name(e: &Entity) -> Option { if e.get_kind() != EntityKind::MemberRefExpr { return None; } @@ -1243,7 +1243,7 @@ fn own_member_name(e: &Entity) -> Option { /// libclang exposes no binary-operator kind, so the operator is the first /// punctuation token that starts at or after the end of the left operand. #[cfg(feature = "libclang")] -fn binary_operator_spelling(node: &Entity) -> Option { +pub(crate) fn binary_operator_spelling(node: &Entity) -> Option { let lhs_end = node .get_children() .first()? @@ -1263,7 +1263,7 @@ fn binary_operator_spelling(node: &Entity) -> Option { /// Is this unary operator an increment or decrement (prefix or postfix)? #[cfg(feature = "libclang")] -fn unary_operator_is_inc_dec(node: &Entity) -> bool { +pub(crate) fn unary_operator_is_inc_dec(node: &Entity) -> bool { let Some(range) = node.get_range() else { return false; }; @@ -1336,7 +1336,7 @@ fn collect_own_members(node: &Entity, out: &mut Vec) { /// wrapper cursors, so recurse for the first node yielding a concrete, /// non-void type name. #[cfg(feature = "libclang")] -fn thrown_type_name(throw: &Entity) -> Option { +pub(crate) fn thrown_type_name(throw: &Entity) -> Option { fn first_typed(e: &Entity) -> Option { if let Some(t) = e.get_type() { let name = bare_type_name(&t.get_display_name()); @@ -1352,7 +1352,7 @@ fn thrown_type_name(throw: &Entity) -> Option { /// The receiver of a method call (`repo_` in `repo_.Save()`), or `None` for an /// implicit-`this` call — the callee reference's own base. #[cfg(feature = "libclang")] -fn call_receiver(call: &Entity) -> Option { +pub(crate) fn call_receiver(call: &Entity) -> Option { let name = call.get_name()?; let callee = call.get_children().into_iter().find(|c| { c.get_kind() == EntityKind::MemberRefExpr && c.get_name() == Some(name.clone()) @@ -1372,7 +1372,7 @@ fn call_receiver(call: &Entity) -> Option { /// `List` / `foo::Bar` → a stable bare type name for the `raises` object. #[cfg(feature = "libclang")] -fn bare_type_name(display: &str) -> String { +pub(crate) fn bare_type_name(display: &str) -> String { let s = display .trim_start_matches("class ") .trim_start_matches("struct "); diff --git a/crates/ruff_cpp_spo/src/events.rs b/crates/ruff_cpp_spo/src/events.rs new file mode 100644 index 00000000000000..3d23c9b4960b82 --- /dev/null +++ b/crates/ruff_cpp_spo/src/events.rs @@ -0,0 +1,1500 @@ +//! Ordered behavioral ore — the sequence the five-set arm throws away. +//! +//! # Why this exists +//! +//! [`crate::walk_tu`]'s body arm ends by collapsing its traversal into sets: +//! `method_body_arm` runs `facts.sort(); facts.dedup();` over all five vectors, +//! unconditionally. And `walk_body` gives `ForStmt` / `WhileStmt` / `DoStmt` / +//! `ForRangeStmt` / `SwitchStmt` / `TryStmt` / `LambdaExpr` no match arm at +//! all — their contents are visited, their boundaries are recorded nowhere. +//! +//! So the shipped representation cannot express *a write inside a loop*, *two +//! writes in a row*, or *a call in the else-branch*. Those are exactly the +//! shapes a behavioral vocabulary would be made of. A sequence learner run +//! against that output is learning words from a bag of letters. +//! +//! This module is a SECOND walk over the same cursor that preserves three +//! things separately: +//! +//! | preserved | where | deduped | +//! |---|---|---| +//! | event identity | [`Symbols`] | yes — the only place | +//! | event order | [`MethodOre::events`] | **never** | +//! | structural context | [`MethodOre::scopes`] | n/a | +//! +//! The same fact occurring twice is two events pointing at one symbol. +//! +//! # Additive by construction +//! +//! Nothing here changes `method_body_arm`, `BodyArm`, `CppMethod`, `expand` +//! or the emitted ndjson. `events_collapse_to_the_shipped_five_sets` asserts +//! that folding these events back into sets reproduces the shipped arm +//! exactly — the change is about what is PRESERVED, not about what a fact is. +//! +//! # Neutrality — the ore must not pre-solve its consumer's experiment +//! +//! The alphabet is 20 primitive kinds. There is deliberately no `RAII`, +//! `IteratorLoop`, `VirtualDispatch` or `Move` kind: a `lock_guard` local is +//! [`EventKind::Decl`] plus a type symbol, `std::move(x)` is +//! [`EventKind::Call`] plus a callee symbol. Those concepts are candidate +//! LEARNED tokens, and emitting them here would be writing a tokenizer's +//! answer into its input. Identities are carried verbatim, never hashed — +//! different consumers hash differently. +//! +//! # What libclang does not give +//! +//! There is **no CFG** in libclang's C API. `control` therefore carries only +//! structurally-certain relations (which construct a condition belongs to, +//! which arm a branch is, and a loop's back edge). Successor/predecessor +//! edges are not available and are never fabricated. Every event also carries +//! its source byte `anchor`, so lexical order stays checkable against +//! traversal order rather than being silently conflated with it. + +use std::collections::BTreeMap; +use std::fmt; +use std::path::Path; + +use clang::source::File; +use clang::{Clang, Entity, EntityKind, Index}; + +use crate::clang_walker::{ + BodyArmConfig, assignment_target, bare_type_name, binary_operator_spelling, call_receiver, + method_body_arm, own_member_name, qualified_name, thrown_type_name, unary_operator_is_inc_dec, +}; + +/// The closed primitive alphabet. Adding a variant is a schema change. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EventKind { + ScopeEnter, + ScopeExit, + Param, + Decl, + CtorInit, + Read, + Write, + ReadWrite, + Condition, + Branch, + Return, + Break, + Continue, + Goto, + Label, + Call, + New, + Delete, + Throw, + Cast, +} + +impl EventKind { + /// The wire spelling. Stable — consumers match on it. + pub fn as_str(self) -> &'static str { + match self { + Self::ScopeEnter => "ScopeEnter", + Self::ScopeExit => "ScopeExit", + Self::Param => "Param", + Self::Decl => "Decl", + Self::CtorInit => "CtorInit", + Self::Read => "Read", + Self::Write => "Write", + Self::ReadWrite => "ReadWrite", + Self::Condition => "Condition", + Self::Branch => "Branch", + Self::Return => "Return", + Self::Break => "Break", + Self::Continue => "Continue", + Self::Goto => "Goto", + Self::Label => "Label", + Self::Call => "Call", + Self::New => "New", + Self::Delete => "Delete", + Self::Throw => "Throw", + Self::Cast => "Cast", + } + } +} + +impl fmt::Display for EventKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The closed scope vocabulary. Class and namespace are deliberately absent: +/// they are constants of a stream, already carried fully-qualified in the +/// method IRI, and emitting them per-method would be redundant structure a +/// tokenizer would then have to learn to ignore. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScopeKind { + Function, + Block, + If, + Then, + Else, + For, + While, + Do, + RangeFor, + Switch, + Case, + Default, + Try, + Catch, + Lambda, +} + +impl ScopeKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Function => "Function", + Self::Block => "Block", + Self::If => "If", + Self::Then => "Then", + Self::Else => "Else", + Self::For => "For", + Self::While => "While", + Self::Do => "Do", + Self::RangeFor => "RangeFor", + Self::Switch => "Switch", + Self::Case => "Case", + Self::Default => "Default", + Self::Try => "Try", + Self::Catch => "Catch", + Self::Lambda => "Lambda", + } + } + + /// Does a loop's exit close a back edge? The one control relation that is + /// structurally certain without a CFG. + fn is_loop(self) -> bool { + matches!(self, Self::For | Self::While | Self::Do | Self::RangeFor) + } +} + +/// What an entity is, for the symbol table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum SymKind { + Member, + Param, + Local, + Callee, + Type, + Scope, + Exception, + Other, +} + +impl SymKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Member => "member", + Self::Param => "param", + Self::Local => "local", + Self::Callee => "callee", + Self::Type => "type", + Self::Scope => "scope_kind", + Self::Exception => "exception", + Self::Other => "other", + } + } +} + +/// Whose entity it is. The distinction the shipped arm cannot make, because it +/// only ever recorded own members. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Role { + Own, + Param, + Local, + This, + Static, + Ext, + Unknown, + None, +} + +impl Role { + pub fn as_str(self) -> &'static str { + match self { + Self::Own => "own", + Self::Param => "param", + Self::Local => "local", + Self::This => "this", + Self::Static => "static", + Self::Ext => "ext", + Self::Unknown => "unknown", + Self::None => "-", + } + } +} + +/// Did a libclang semantic API answer this, or did the walker infer it from +/// cursor shape and tokens? A consumer that reports a result must be able to +/// say how much of what it consumed the compiler simply handed over. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Prov { + Clang, + Walk, +} + +impl Prov { + pub fn as_str(self) -> &'static str { + match self { + Self::Clang => "clang", + Self::Walk => "walk", + } + } +} + +/// One occurrence. Never deduped, never sorted. +#[derive(Debug, Clone)] +pub struct OreEvent { + pub seq: u32, + pub kind: EventKind, + pub subj: Option, + pub obj: Option, + pub scope: u32, + pub parent: Option, + pub anchor: u32, + pub control: Option, + pub type_rel: Option, + pub prov: Prov, +} + +/// One scope in a method's tree. +#[derive(Debug, Clone)] +pub struct OreScope { + pub id: u32, + pub parent: Option, + pub kind: ScopeKind, + pub depth: u16, + pub enter_seq: u32, + pub exit_seq: u32, +} + +/// One method's ore, plus the header facts a consumer joins on. +#[derive(Debug, Clone)] +#[expect( + clippy::struct_excessive_bools, + reason = "independent C++ method qualifiers, mirroring `ruff_spo_triplet::CppMethod`'s own \ + carve-out — any combination is valid, so two-variant enums would be artificial" +)] +pub struct MethodOre { + pub iri: String, + pub tu: String, + pub class: String, + pub events: Vec, + pub scopes: Vec, + pub is_const: bool, + pub is_static: bool, + pub is_virtual: bool, + pub overrides: bool, + pub mkind: &'static str, + pub access: &'static str, + pub n_params: u32, + /// The SHIPPED five-set arm's view of the very same cursor, and the + /// centroid `recipe::classify` derives from it. Carried here so a + /// consumer can compare the ordered ore against A0 without a second + /// parse — and, more importantly, without a second WALK, which would + /// make any difference between them ambiguous. + pub set_reads: u32, + pub set_writes: u32, + pub set_raises: u32, + pub set_calls: u32, + pub set_guarded: u32, + pub centroid: String, +} + +/// The canonical identity table — the ONLY place dedup happens. +#[derive(Debug, Default)] +pub struct Symbols { + index: BTreeMap<(SymKind, Role, String), u32>, + rows: Vec<(SymKind, Role, String, Prov)>, +} + +impl Symbols { + pub fn intern(&mut self, kind: SymKind, role: Role, name: &str, prov: Prov) -> String { + let key = (kind, role, name.to_string()); + if let Some(id) = self.index.get(&key) { + return format!("s{id}"); + } + let id = u32::try_from(self.rows.len()).unwrap_or(u32::MAX); + self.index.insert(key, id); + self.rows.push((kind, role, name.to_string(), prov)); + format!("s{id}") + } + + pub fn rows(&self) -> impl Iterator { + self.rows + .iter() + .enumerate() + .map(|(i, (k, r, n, p))| (format!("s{i}"), *k, *r, n.as_str(), *p)) + } + + pub fn len(&self) -> usize { + self.rows.len() + } + + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } +} + +/// Everything one translation unit yielded. +#[derive(Debug, Default)] +pub struct TuOre { + pub methods: Vec, + pub diagnostics: Vec, +} + +// ───────────────────────────────────────────────────────────────────────── +// The walk +// ───────────────────────────────────────────────────────────────────────── + +struct Walk<'s> { + syms: &'s mut Symbols, + events: Vec, + scopes: Vec, + stack: Vec, +} + +impl<'s> Walk<'s> { + fn new(syms: &'s mut Symbols) -> Self { + Self { + syms, + events: Vec::new(), + scopes: Vec::new(), + stack: Vec::new(), + } + } + + fn cur(&self) -> u32 { + self.stack.last().copied().unwrap_or(0) + } + + fn parent(&self) -> Option { + let n = self.stack.len(); + (n >= 2).then(|| self.stack[n - 2]) + } + + fn anchor(e: &Entity<'_>) -> u32 { + e.get_range() + .map(|r| r.get_start().get_file_location().offset) + .unwrap_or(0) + } + + fn push(&mut self, kind: EventKind, e: &Entity<'_>) -> usize { + let seq = u32::try_from(self.events.len()).unwrap_or(u32::MAX); + self.events.push(OreEvent { + seq, + kind, + subj: None, + obj: None, + scope: self.cur(), + parent: self.parent(), + anchor: Self::anchor(e), + control: None, + type_rel: None, + prov: Prov::Walk, + }); + self.events.len() - 1 + } + + /// Open a scope, run `body`, close it. Enter/exit carry the scope's own id + /// and its kind symbol, so a nested sequence cannot flatten into a sibling. + fn scoped(&mut self, kind: ScopeKind, e: &Entity<'_>, body: F) { + let id = u32::try_from(self.scopes.len()).unwrap_or(u32::MAX); + let depth = u16::try_from(self.stack.len()).unwrap_or(u16::MAX); + let parent = self.stack.last().copied(); + let ksym = self + .syms + .intern(SymKind::Scope, Role::None, kind.as_str(), Prov::Walk); + self.stack.push(id); + let enter = self.push(EventKind::ScopeEnter, e); + self.events[enter].subj = Some(ksym.clone()); + let enter_seq = self.events[enter].seq; + self.scopes.push(OreScope { + id, + parent, + kind, + depth, + enter_seq, + exit_seq: enter_seq, + }); + body(self); + let exit = self.push(EventKind::ScopeExit, e); + self.events[exit].subj = Some(ksym); + if kind.is_loop() { + self.events[exit].control = Some("back_edge".to_string()); + } + let exit_seq = self.events[exit].seq; + if let Some(s) = self.scopes.iter_mut().find(|s| s.id == id) { + s.exit_seq = exit_seq; + } + self.stack.pop(); + } + + fn children(&mut self, e: &Entity<'_>) { + for c in e.get_children() { + self.node(&c); + } + } + + /// A construct's body. A `CompoundStmt` here is the braces of the + /// construct we already opened a scope for, so it is walked THROUGH — + /// otherwise `if (x) f();` and `if (x) { f(); }` would differ by a scope, + /// and a tokenizer would learn brace style instead of behavior. + fn body(&mut self, e: &Entity<'_>) { + if e.get_kind() == EntityKind::CompoundStmt { + self.children(e); + } else { + self.node(e); + } + } + + /// The role of a referenced declaration. `prov=clang` — libclang resolved it. + fn ref_role(e: &Entity<'_>) -> (Role, SymKind, Prov) { + match e.get_reference().map(|r| r.get_kind()) { + Some(EntityKind::ParmDecl) => (Role::Param, SymKind::Param, Prov::Clang), + Some(EntityKind::VarDecl) => (Role::Local, SymKind::Local, Prov::Clang), + Some(EntityKind::FieldDecl) => (Role::Own, SymKind::Member, Prov::Clang), + _ => (Role::Unknown, SymKind::Other, Prov::Walk), + } + } + + fn node(&mut self, e: &Entity<'_>) { + match e.get_kind() { + EntityKind::CompoundStmt => self.scoped(ScopeKind::Block, e, |w| w.children(e)), + + EntityKind::IfStmt => self.scoped(ScopeKind::If, e, |w| { + let ch = e.get_children(); + // [cond, then, else?] — the arms are taken by POSITION, so a + // braced and an unbraced arm give the identical scope tree. + if let Some(cond) = ch.first() { + w.node(cond); + let i = w.push(EventKind::Condition, cond); + w.events[i].control = Some("if".to_string()); + } + if let Some(then) = ch.get(1) { + let i = w.push(EventKind::Branch, then); + w.events[i].control = Some("then".to_string()); + w.scoped(ScopeKind::Then, then, |w| w.body(then)); + } + if let Some(els) = ch.get(2) { + let i = w.push(EventKind::Branch, els); + w.events[i].control = Some("else".to_string()); + w.scoped(ScopeKind::Else, els, |w| w.body(els)); + } + }), + + EntityKind::ForStmt + | EntityKind::WhileStmt + | EntityKind::DoStmt + | EntityKind::ForRangeStmt => { + let (sk, tag) = match e.get_kind() { + EntityKind::ForStmt => (ScopeKind::For, "for"), + EntityKind::WhileStmt => (ScopeKind::While, "while"), + EntityKind::DoStmt => (ScopeKind::Do, "do"), + _ => (ScopeKind::RangeFor, "rangefor"), + }; + let is_do = sk == ScopeKind::Do; + self.scoped(sk, e, |w| { + // libclang labels no sub-part of a loop header, and a + // `for(;;)` simply has fewer children — so for `for` / + // `while` / range-`for` the LAST child is the body and + // everything before it is the header. `do { } while (c)` + // is the exception in BOTH respects: its body comes + // FIRST and its condition is evaluated after the body, so + // treating the last child as the body would walk the + // condition as a statement and open a spurious Block + // around the real one. The header's parts are not + // individually distinguished; `anchor` keeps their + // source positions. + let ch = e.get_children(); + if is_do { + if let Some((body, rest)) = ch.split_first() { + w.body(body); + for c in rest { + w.node(c); + } + } + let i = w.push(EventKind::Condition, e); + w.events[i].control = Some(tag.to_string()); + } else { + let (head, body) = ch.split_at(ch.len().saturating_sub(1)); + for c in head { + w.node(c); + } + let i = w.push(EventKind::Condition, e); + w.events[i].control = Some(tag.to_string()); + for c in body { + w.body(c); + } + } + }); + } + + EntityKind::SwitchStmt => self.scoped(ScopeKind::Switch, e, |w| { + let ch = e.get_children(); + if let Some(cond) = ch.first() { + w.node(cond); + let i = w.push(EventKind::Condition, cond); + w.events[i].control = Some("switch".to_string()); + } + for c in ch.iter().skip(1) { + w.body(c); + } + }), + + EntityKind::CaseStmt | EntityKind::DefaultStmt => { + let is_case = e.get_kind() == EntityKind::CaseStmt; + let i = self.push(EventKind::Branch, e); + self.events[i].control = Some(if is_case { "case" } else { "default" }.to_string()); + let sk = if is_case { + ScopeKind::Case + } else { + ScopeKind::Default + }; + self.scoped(sk, e, |w| { + for c in e.get_children() { + w.body(&c); + } + }); + } + + EntityKind::TryStmt => self.scoped(ScopeKind::Try, e, |w| { + for c in e.get_children() { + if c.get_kind() == EntityKind::CatchStmt { + let i = w.push(EventKind::Branch, &c); + w.events[i].control = Some("catch".to_string()); + w.scoped(ScopeKind::Catch, &c, |w| { + for g in c.get_children() { + w.body(&g); + } + }); + } else { + w.body(&c); + } + } + }), + + EntityKind::CatchStmt => { + let i = self.push(EventKind::Branch, e); + self.events[i].control = Some("catch".to_string()); + self.scoped(ScopeKind::Catch, e, |w| { + for c in e.get_children() { + w.body(&c); + } + }); + } + + EntityKind::LambdaExpr => self.scoped(ScopeKind::Lambda, e, |w| { + for c in e.get_children() { + w.body(&c); + } + }), + + EntityKind::ReturnStmt => { + self.push(EventKind::Return, e); + self.children(e); + } + EntityKind::BreakStmt => { + self.push(EventKind::Break, e); + } + EntityKind::ContinueStmt => { + self.push(EventKind::Continue, e); + } + EntityKind::GotoStmt | EntityKind::IndirectGotoStmt => { + self.push(EventKind::Goto, e); + } + EntityKind::LabelStmt => { + self.push(EventKind::Label, e); + self.children(e); + } + + EntityKind::VarDecl => { + let name = e.get_name().unwrap_or_default(); + let sym = self + .syms + .intern(SymKind::Local, Role::Local, &name, Prov::Clang); + let ty = e + .get_type() + .map(|t| bare_type_name(&t.get_display_name())) + .filter(|t| !t.is_empty()) + .map(|t| self.syms.intern(SymKind::Type, Role::None, &t, Prov::Clang)); + let i = self.push(EventKind::Decl, e); + self.events[i].subj = Some(sym); + self.events[i].type_rel = ty; + self.events[i].prov = Prov::Clang; + self.children(e); + } + + EntityKind::MemberRef => { + // A constructor's member-initializer target. The shipped arm + // never sees these, so every ctor field initialisation is + // invisible to it. + if let Some(name) = e.get_name() { + let sym = self + .syms + .intern(SymKind::Member, Role::Own, &name, Prov::Clang); + let i = self.push(EventKind::CtorInit, e); + self.events[i].subj = Some(sym); + self.events[i].prov = Prov::Clang; + } + self.children(e); + } + + EntityKind::ThrowExpr => { + let ty = thrown_type_name(e).map(|t| { + self.syms + .intern(SymKind::Exception, Role::None, &t, Prov::Clang) + }); + let i = self.push(EventKind::Throw, e); + self.events[i].obj = ty; + self.children(e); + } + + EntityKind::NewExpr | EntityKind::DeleteExpr => { + let is_new = e.get_kind() == EntityKind::NewExpr; + let ty = e + .get_type() + .map(|t| bare_type_name(&t.get_display_name())) + .filter(|t| !t.is_empty()) + .map(|t| self.syms.intern(SymKind::Type, Role::None, &t, Prov::Clang)); + let i = self.push( + if is_new { + EventKind::New + } else { + EventKind::Delete + }, + e, + ); + self.events[i].obj = ty; + self.children(e); + } + + EntityKind::CStyleCastExpr + | EntityKind::StaticCastExpr + | EntityKind::DynamicCastExpr + | EntityKind::ReinterpretCastExpr + | EntityKind::ConstCastExpr + | EntityKind::FunctionalCastExpr => { + let tag = match e.get_kind() { + EntityKind::CStyleCastExpr => "c", + EntityKind::StaticCastExpr => "static", + EntityKind::DynamicCastExpr => "dynamic", + EntityKind::ReinterpretCastExpr => "reinterpret", + EntityKind::ConstCastExpr => "const", + _ => "functional", + }; + let i = self.push(EventKind::Cast, e); + self.events[i].type_rel = Some(tag.to_string()); + self.children(e); + } + + EntityKind::BinaryOperator => { + let ch = e.get_children(); + if let Some((lhs, rest)) = ch.split_first() + && binary_operator_spelling(e).as_deref() == Some("=") + { + self.write_event(lhs, EventKind::Write); + self.node_skipping_top_member(lhs); + for r in rest { + self.node(r); + } + } else { + self.children(e); + } + } + + EntityKind::CompoundAssignOperator => { + let ch = e.get_children(); + if let Some((lhs, rest)) = ch.split_first() { + self.write_event(lhs, EventKind::ReadWrite); + self.node_skipping_top_member(lhs); + for r in rest { + self.node(r); + } + } + } + + EntityKind::UnaryOperator => { + let ch = e.get_children(); + if let Some(op) = ch.first() + && unary_operator_is_inc_dec(e) + { + self.write_event(op, EventKind::ReadWrite); + self.node_skipping_top_member(op); + } else { + self.children(e); + } + } + + EntityKind::CallExpr => self.call(e), + + EntityKind::MemberRefExpr => { + let (sym, prov) = if let Some(m) = own_member_name(e) { + ( + self.syms.intern(SymKind::Member, Role::Own, &m, Prov::Walk), + Prov::Walk, + ) + } else { + let name = e.get_name().unwrap_or_default(); + ( + self.syms + .intern(SymKind::Member, Role::Ext, &name, Prov::Walk), + Prov::Walk, + ) + }; + let i = self.push(EventKind::Read, e); + self.events[i].subj = Some(sym); + self.events[i].prov = prov; + self.children(e); + } + + EntityKind::DeclRefExpr => { + let (role, kind, prov) = Self::ref_role(e); + if matches!(role, Role::Param | Role::Local) { + let name = e.get_name().unwrap_or_default(); + let sym = self.syms.intern(kind, role, &name, prov); + let i = self.push(EventKind::Read, e); + self.events[i].subj = Some(sym); + self.events[i].prov = prov; + } + self.children(e); + } + + _ => self.children(e), + } + } + + /// Record a write/read-modify-write against whatever the LHS names. + fn write_event(&mut self, lhs: &Entity<'_>, kind: EventKind) { + let (sym, prov) = if let Some(m) = assignment_target(lhs) { + ( + self.syms.intern(SymKind::Member, Role::Own, &m, Prov::Walk), + Prov::Walk, + ) + } else { + let (role, sk, prov) = Self::ref_role(lhs); + let name = lhs.get_name().unwrap_or_else(|| "?".to_string()); + (self.syms.intern(sk, role, &name, prov), prov) + }; + let i = self.push(kind, lhs); + self.events[i].subj = Some(sym); + self.events[i].prov = prov; + } + + /// Walk an assignment target's children WITHOUT re-recording the target + /// itself as a read — mirroring the shipped arm, which walks the LHS with + /// `walk_body` precisely to skip its own top-level `MemberRefExpr`. + fn node_skipping_top_member(&mut self, lhs: &Entity<'_>) { + self.children(lhs); + } + + fn call(&mut self, e: &Entity<'_>) { + let name = e.get_name().unwrap_or_default(); + let referenced = e.get_reference(); + let (tag, prov) = referenced.map_or(("-", Prov::Walk), |r| { + let t = if r.is_virtual_method() { + "virtual" + } else if r.is_static_method() { + "static" + } else { + match r.get_kind() { + EntityKind::Constructor => "ctor", + EntityKind::Destructor => "dtor", + EntityKind::Method | EntityKind::ConversionFunction => "member", + EntityKind::FunctionDecl => "free", + _ => "-", + } + }; + (t, Prov::Clang) + }); + let recv = call_receiver(e); + let recv_sym = recv + .as_ref() + .map(|r| self.syms.intern(SymKind::Callee, Role::Ext, r, Prov::Walk)); + let callee = self.syms.intern( + SymKind::Callee, + if recv.is_some() { + Role::Ext + } else { + Role::This + }, + &name, + prov, + ); + let i = self.push(EventKind::Call, e); + self.events[i].subj = recv_sym; + self.events[i].obj = Some(callee); + self.events[i].type_rel = Some(tag.to_string()); + self.events[i].prov = prov; + // Walk the arguments and the RECEIVER, but never the callee reference + // itself: `foo()` nests a `MemberRefExpr` named `foo`, and recording + // it would emit a read of a member that does not exist. The shipped + // walker names this trap; this one fell into it once. + for c in e.get_children() { + let is_callee = c.get_kind() == EntityKind::MemberRefExpr + && c.get_name().as_deref() == Some(name.as_str()); + if is_callee { + self.children(&c); // the receiver base still counts + } else { + self.node(&c); + } + } + } +} + +// ───────────────────────────────────────────────────────────────────────── +// Entry points +// ───────────────────────────────────────────────────────────────────────── + +/// The method IRI, byte-identical to what `ruff_spo_triplet::expand` builds +/// for the same method — the join key every consumer relies on. +fn method_iri(m: &Entity<'_>) -> Option { + let name = m.get_name()?; + let parent = m.get_semantic_parent()?; + let params: Vec = m + .get_arguments() + .into_iter() + .flatten() + .filter_map(|a| a.get_type().map(|t| t.get_display_name())) + .collect(); + let refq = m + .get_type() + .and_then(|t| t.get_ref_qualifier()) + .map(|q| match q { + clang::RefQualifier::LValue => " &", + clang::RefQualifier::RValue => " &&", + }) + .unwrap_or(""); + Some(format!( + "{}.{name}({}){}{}", + qualified_name(&parent), + params.join(","), + if m.is_const_method() { " const" } else { "" }, + refq + )) +} + +fn is_callable(k: EntityKind) -> bool { + matches!( + k, + EntityKind::Method + | EntityKind::Constructor + | EntityKind::Destructor + | EntityKind::ConversionFunction + | EntityKind::FunctionDecl + ) +} + +fn collect( + e: &Entity<'_>, + tu: &str, + syms: &mut Symbols, + out: &mut Vec, + main: Option>, + arm_cfg: &BodyArmConfig, +) { + for c in e.get_children() { + if is_callable(c.get_kind()) && c.is_definition() { + // Only bodies from THIS translation unit's own file, or every + // header a TU pulls in is re-harvested once per includer and the + // corpus silently multiplies. + let same = c + .get_location() + .map(|l| l.get_file_location().file) + .is_some_and(|f| f.is_some() && f == main); + if same && let Some(iri) = method_iri(&c) { + out.push(method_ore(&c, tu, &iri, syms, arm_cfg)); + } + } + collect(&c, tu, syms, out, main, arm_cfg); + } +} + +fn method_ore( + m: &Entity<'_>, + tu: &str, + iri: &str, + syms: &mut Symbols, + arm_cfg: &BodyArmConfig, +) -> MethodOre { + // The shipped arm, on this exact cursor, in this exact parse. + let arm = method_body_arm(m, arm_cfg); + let shipped = ruff_spo_triplet::CppMethod { + name: m.get_name().unwrap_or_default(), + writes: arm.writes.clone(), + reads: arm.reads.clone(), + raises: arm.raises.clone(), + calls: arm.calls.clone(), + guarded_writes: arm.guarded_writes.clone(), + ..Default::default() + }; + let centroid = format!("{:?}", ruff_spo_triplet::classify(&shipped)); + let mut w = Walk::new(syms); + let anchor = m; + w.scoped(ScopeKind::Function, anchor, |w| { + for a in m.get_arguments().into_iter().flatten() { + let name = a.get_name().unwrap_or_default(); + let sym = w + .syms + .intern(SymKind::Param, Role::Param, &name, Prov::Clang); + let ty = a + .get_type() + .map(|t| bare_type_name(&t.get_display_name())) + .filter(|t| !t.is_empty()) + .map(|t| w.syms.intern(SymKind::Type, Role::None, &t, Prov::Clang)); + let i = w.push(EventKind::Param, &a); + w.events[i].subj = Some(sym); + w.events[i].type_rel = ty; + w.events[i].prov = Prov::Clang; + } + for c in m.get_children() { + match c.get_kind() { + // Parameters are emitted above, in declaration order. + EntityKind::ParmDecl => {} + // The function's own CompoundStmt would open a redundant Block + // scope inside Function; walk through it. + EntityKind::CompoundStmt => w.children(&c), + _ => w.node(&c), + } + } + }); + let class = m + .get_semantic_parent() + .map(|p| qualified_name(&p)) + .unwrap_or_default(); + MethodOre { + iri: iri.to_string(), + tu: tu.to_string(), + class, + events: w.events, + scopes: w.scopes, + is_const: m.is_const_method(), + is_static: m.is_static_method(), + is_virtual: m.is_virtual_method(), + overrides: m.get_overridden_methods().is_some_and(|o| !o.is_empty()), + mkind: match m.get_kind() { + EntityKind::Constructor => "ctor", + EntityKind::Destructor => "dtor", + EntityKind::ConversionFunction => "operator", + _ => { + if m.get_name().is_some_and(|n| n.starts_with("operator")) { + "operator" + } else { + "method" + } + } + }, + access: match m.get_accessibility() { + Some(clang::Accessibility::Protected) => "protected", + Some(clang::Accessibility::Private) => "private", + _ => "public", + }, + n_params: u32::try_from(m.get_arguments().map_or(0, |a| a.len())).unwrap_or(0), + set_reads: u32::try_from(arm.reads.len()).unwrap_or(0), + set_writes: u32::try_from(arm.writes.len()).unwrap_or(0), + set_raises: u32::try_from(arm.raises.len()).unwrap_or(0), + set_calls: u32::try_from(arm.calls.len()).unwrap_or(0), + set_guarded: u32::try_from(arm.guarded_writes.len()).unwrap_or(0), + centroid, + } +} + +/// Walk ONE translation unit and return every method definition's ordered ore. +/// +/// # Errors +/// +/// [`crate::WalkError::Libclang`] if libclang fails to initialise; +/// [`crate::WalkError::Parse`] if the TU fails to parse. Parse DIAGNOSTICS are +/// returned rather than swallowed — a partial AST must be visible, for the +/// same reason `walk_tu_with_diagnostics` exists. +pub fn walk_tu_events( + path: &Path, + args: &[String], + syms: &mut Symbols, +) -> Result { + let clang = Clang::new().map_err(crate::WalkError::Libclang)?; + let index = Index::new(&clang, false, false); + let tu = index + .parser(path) + .arguments(args) + .skip_function_bodies(false) + .parse() + .map_err(|e| crate::WalkError::Parse(e.to_string()))?; + let diagnostics = tu + .get_diagnostics() + .into_iter() + .filter(|d| d.get_severity() >= clang::diagnostic::Severity::Error) + .map(|d| d.formatter().format()) + .collect(); + let mut methods = Vec::new(); + let main = tu.get_file(path); + let arm_cfg = BodyArmConfig::default(); + collect( + &tu.get_entity(), + &path.display().to_string(), + syms, + &mut methods, + main, + &arm_cfg, + ); + Ok(TuOre { + methods, + diagnostics, + }) +} + +#[cfg(all(test, feature = "libclang"))] +mod tests { + use std::io::Write; + + use super::*; + use crate::clang_walker::CLANG_TEST_LOCK; + + /// One parse per fixture. `Clang` is a process singleton, so every test + /// here serialises on the crate-wide lock — the same lock `arm_tests` and + /// `libclang_tests` take. + fn ore(name: &str, src: &str) -> (Vec, Symbols) { + let dir = std::env::temp_dir().join(format!("cpp_ore_{name}")); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("f.cpp"); + let mut fh = std::fs::File::create(&path).expect("fixture file"); + fh.write_all(src.as_bytes()).expect("write fixture"); + drop(fh); + let _guard = CLANG_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut syms = Symbols::default(); + let tu = walk_tu_events(&path, &["-std=c++17".to_string()], &mut syms).expect("walk"); + (tu.methods, syms) + } + + fn find<'a>(ms: &'a [MethodOre], needle: &str) -> &'a MethodOre { + ms.iter() + .find(|m| m.iri.contains(needle)) + .unwrap_or_else(|| { + panic!( + "no method matching {needle} in {:?}", + ms.iter().map(|m| &m.iri).collect::>() + ) + }) + } + + fn kinds(m: &MethodOre) -> Vec<&'static str> { + m.events.iter().map(|e| e.kind.as_str()).collect() + } + + fn scope_kinds(m: &MethodOre) -> Vec<&'static str> { + m.scopes.iter().map(|s| s.kind.as_str()).collect() + } + + /// THE CONTROL. Its expected sequence was hand-written before this code + /// existed (`EXPECTED-run.md`); where the two disagreed, the disagreement + /// was recorded rather than reconciled, and this test pins what libclang + /// actually does. + #[test] + fn the_control_loop_over_a_branch_emits_its_whole_structure() { + let (ms, syms) = ore( + "control", + r" +struct C { + void run(int n); + void foo(); + void bar(); + int count_; +}; +void C::run(int n) { + for (int i = 0; i < n; ++i) { + if (count_ > 0) { foo(); } else { bar(); } + } +}", + ); + let m = find(&ms, "C.run(int)"); + assert_eq!( + kinds(m), + vec![ + "ScopeEnter", + "Param", + "ScopeEnter", + "Decl", + "Read", + "Read", + "ReadWrite", + "Condition", + "ScopeEnter", + "Read", + "Condition", + "Branch", + "ScopeEnter", + "Call", + "ScopeExit", + "Branch", + "ScopeEnter", + "Call", + "ScopeExit", + "ScopeExit", + "ScopeExit", + "ScopeExit", + ], + "the control's event sequence" + ); + assert_eq!( + scope_kinds(m), + vec!["Function", "For", "If", "Then", "Else"], + "braces must not add a scope: For > If > {{Then, Else}}" + ); + // The loop's exit closes a back edge — the one control relation that + // is structurally certain without a CFG. + let exit_for = m + .events + .iter() + .find(|e| e.kind == EventKind::ScopeExit && e.control.as_deref() == Some("back_edge")); + assert!(exit_for.is_some(), "the loop exit must carry its back edge"); + // A call's callee must NOT also appear as a member read. + let reads: Vec<&str> = m + .events + .iter() + .filter(|e| e.kind == EventKind::Read) + .filter_map(|e| e.subj.as_deref()) + .filter_map(|s| syms.rows().find(|r| r.0 == s).map(|r| r.3)) + .map(str::to_string) + .map(|s| Box::leak(s.into_boxed_str()) as &str) + .collect(); + assert!( + !reads.contains(&"foo") && !reads.contains(&"bar"), + "a call's own callee is a MemberRefExpr; it must never read as a member: {reads:?}" + ); + } + + /// Braces are style, not behavior: the two forms must give one tree. + #[test] + fn a_braced_and_an_unbraced_arm_give_the_identical_stream() { + let (ms, _) = ore( + "braces", + r" +struct C { void a(int n); void b(int n); int x_; }; +void C::a(int n) { if (n > 0) { x_ = n; } else { x_ = 0; } } +void C::b(int n) { if (n > 0) x_ = n; else x_ = 0; }", + ); + assert_eq!( + kinds(find(&ms, "C.a(int)")), + kinds(find(&ms, "C.b(int)")), + "brace style must not change the event stream" + ); + assert_eq!( + scope_kinds(find(&ms, "C.a(int)")), + scope_kinds(find(&ms, "C.b(int)")) + ); + } + + /// Every scope kind the schema names, on one fixture. + #[test] + fn every_scope_kind_appears_with_the_right_parent_and_depth() { + let (ms, _) = ore( + "scopes", + r" +#include +struct C { + void loops(const std::vector& xs, int n); + void branches(int n); + void protect(); + void lam(int n); + void blk(int n); + int x_; +}; +void C::loops(const std::vector& xs, int n) { + for (int i = 0; i < n; ++i) { x_ += i; } + while (x_ < n) { ++x_; } + do { --x_; } while (x_ > n); + for (int v : xs) { x_ += v; } +} +void C::branches(int n) { + switch (n) { case 0: x_ = 1; break; default: x_ = 2; } +} +void C::protect() { try { x_ = 1; } catch (const int& e) { x_ = 2; } } +void C::lam(int n) { auto f = [this, n]() { x_ = n; }; f(); } +void C::blk(int n) { { x_ = n; } }", + ); + let have = |m: &MethodOre, k: &str| scope_kinds(m).contains(&k); + let loops = find(&ms, "C.loops"); + for k in ["For", "While", "Do", "RangeFor"] { + assert!(have(loops, k), "missing {k}"); + } + let br = find(&ms, "C.branches"); + for k in ["Switch", "Case", "Default"] { + assert!(have(br, k), "missing {k}"); + } + let pr = find(&ms, "C.protect"); + assert!(have(pr, "Try") && have(pr, "Catch")); + assert!(have(find(&ms, "C.lam"), "Lambda")); + // A FREE-STANDING compound is a real Block; a construct's body is not. + assert!(have(find(&ms, "C.blk"), "Block")); + assert!( + !have(loops, "Block"), + "a loop body's braces must not open a Block" + ); + // Depth and parentage, not just presence. + let f = &loops.scopes[0]; + assert_eq!((f.kind, f.depth, f.parent), (ScopeKind::Function, 0, None)); + for s in loops.scopes.iter().filter(|s| s.kind.is_loop()) { + assert_eq!( + s.parent, + Some(0), + "a top-level loop's parent is the function" + ); + assert_eq!(s.depth, 1); + } + } + + /// Nested and sibling scopes must stay distinguishable — the property that + /// makes a sequence recoverable rather than flattened. + #[test] + fn sibling_and_nested_scopes_do_not_flatten_into_each_other() { + let (ms, _) = ore( + "nesting", + r" +struct C { void f(int n); int x_; }; +void C::f(int n) { + for (int i = 0; i < n; ++i) { + if (n > 1) { x_ = 1; } + if (n > 2) { x_ = 2; } + for (int j = 0; j < n; ++j) { x_ = 3; } + } +}", + ); + let m = find(&ms, "C.f(int)"); + let ifs: Vec<&OreScope> = m + .scopes + .iter() + .filter(|s| s.kind == ScopeKind::If) + .collect(); + assert_eq!(ifs.len(), 2, "two sibling ifs"); + assert_ne!(ifs[0].id, ifs[1].id, "siblings need distinct ids"); + assert_eq!(ifs[0].parent, ifs[1].parent, "siblings share a parent"); + assert!( + ifs[0].exit_seq < ifs[1].enter_seq, + "siblings do not interleave" + ); + let fors: Vec<&OreScope> = m + .scopes + .iter() + .filter(|s| s.kind == ScopeKind::For) + .collect(); + assert_eq!(fors.len(), 2); + assert_eq!(fors[0].depth, 1); + assert_eq!(fors[1].depth, 2, "the inner loop is one level deeper"); + assert_eq!(fors[1].parent, Some(fors[0].id)); + } + + /// The whole point: an occurrence is not an identity. + #[test] + fn two_writes_to_one_member_are_two_events_and_one_symbol() { + let (ms, syms) = ore( + "dupes", + r" +struct C { void f(int n); int x_; }; +void C::f(int n) { x_ = n; x_ = n + 1; }", + ); + let m = find(&ms, "C.f(int)"); + let writes: Vec<&OreEvent> = m + .events + .iter() + .filter(|e| e.kind == EventKind::Write) + .collect(); + assert_eq!(writes.len(), 2, "two occurrences survive"); + assert_eq!( + writes[0].subj, writes[1].subj, + "both point at ONE canonical symbol" + ); + assert_eq!( + syms.rows().filter(|r| r.3 == "x_").count(), + 1, + "the symbol table deduped" + ); + // And A0, on the same cursor, kept one. + assert_eq!( + m.set_writes, 1, + "the shipped arm collapses both to one fact" + ); + } + + /// The additive guarantee: folding the events back into sets must + /// reproduce the shipped arm exactly. This is what proves the change is + /// about what is PRESERVED, not about what a fact is. + #[test] + fn events_collapse_to_the_shipped_five_sets() { + let (ms, syms) = ore( + "collapse", + r" +struct E {}; +struct C { + void f(int n); + int x_; int y_; +}; +void C::f(int n) { + x_ = n; + x_ = n + 1; + y_ += x_; + if (n > 2) { throw E(); } +}", + ); + let m = find(&ms, "C.f(int)"); + let name_of = |id: &str| { + syms.rows() + .find(|r| r.0 == id) + .map(|r| (r.2, r.3.to_string())) + }; + let mut reads: Vec = Vec::new(); + let mut writes: Vec = Vec::new(); + let mut raises: Vec = Vec::new(); + for e in &m.events { + match e.kind { + EventKind::Read | EventKind::ReadWrite => { + if let Some(id) = &e.subj + && let Some((Role::Own, n)) = name_of(id) + { + reads.push(n); + } + } + EventKind::Throw => { + if let Some(id) = &e.obj + && let Some((_, n)) = name_of(id) + { + raises.push(n); + } + } + _ => {} + } + if matches!(e.kind, EventKind::Write | EventKind::ReadWrite) + && let Some(id) = &e.subj + && let Some((Role::Own, n)) = name_of(id) + { + writes.push(n); + } + } + for v in [&mut reads, &mut writes, &mut raises] { + v.sort(); + v.dedup(); + } + assert_eq!( + ( + u32::try_from(writes.len()).unwrap_or(u32::MAX), + u32::try_from(raises.len()).unwrap_or(u32::MAX) + ), + (m.set_writes, m.set_raises), + "collapsing the ore must reproduce the shipped arm's sets" + ); + assert_eq!( + u32::try_from(reads.len()).unwrap_or(u32::MAX), + m.set_reads, + "reads agree too" + ); + } + + /// Class and namespace are constants of a stream, not events. + #[test] + fn the_class_and_namespace_ride_the_iri_and_never_become_scopes() { + let (ms, _) = ore( + "ns", + r" +namespace outer { namespace inner { +struct C { void f(); int x_; }; +void C::f() { x_ = 1; } +} }", + ); + let m = find(&ms, "C.f()"); + assert!( + m.iri.contains("outer::inner::C"), + "iri carries the path: {}", + m.iri + ); + assert_eq!(m.class, "outer::inner::C"); + assert!( + m.scopes + .iter() + .all(|s| s.kind != ScopeKind::Block || s.depth > 0), + "no class/namespace scope is emitted" + ); + } + + /// Anchors are the lexical truth against which traversal order is checked. + /// Loop headers are the known exception and this pins it rather than + /// asserting a property that does not hold. + #[test] + fn anchors_are_non_decreasing_except_across_a_loop_header() { + let (ms, _) = ore( + "anchors", + r" +struct C { void f(int n); int x_; }; +void C::f(int n) { x_ = n; if (n > 0) { x_ = 1; } }", + ); + let m = find(&ms, "C.f(int)"); + let mut last = 0; + for e in &m.events { + if e.kind == EventKind::ScopeExit { + continue; // an exit re-uses its construct's start anchor + } + assert!( + e.anchor >= last, + "anchor went backwards at seq {} ({})", + e.seq, + e.kind + ); + last = e.anchor; + } + } + + /// A constructor's member-initializer list is invisible to the shipped + /// arm; the ore records it. + #[test] + fn a_constructor_initializer_list_becomes_ctorinit_events() { + let (ms, _) = ore( + "ctorinit", + r" +struct C { C(int s); int a_; int b_; }; +C::C(int s) : a_(0), b_(s) {}", + ); + let m = find(&ms, "C.C(int)"); + assert_eq!( + m.events + .iter() + .filter(|e| e.kind == EventKind::CtorInit) + .count(), + 2, + "both initialisers" + ); + assert_eq!(m.mkind, "ctor"); + } + + /// A virtual call is labelled by libclang, not guessed — and the label is + /// marked `clang` so a consumer can report how much the compiler gave it. + #[test] + fn a_virtual_call_is_labelled_by_libclang_and_marked_as_such() { + let (ms, _) = ore( + "virt", + r" +struct B { virtual void v(); void nv(); virtual ~B(); }; +struct C : B { void f(B* p); }; +void C::f(B* p) { p->v(); p->nv(); }", + ); + let m = find(&ms, "C.f(B *)"); + let calls: Vec<(&str, Prov)> = m + .events + .iter() + .filter(|e| e.kind == EventKind::Call) + .map(|e| (e.type_rel.as_deref().unwrap_or("-"), e.prov)) + .collect(); + assert!( + calls.contains(&("virtual", Prov::Clang)), + "the virtual call must be labelled by clang: {calls:?}" + ); + assert!( + calls.contains(&("member", Prov::Clang)), + "and the non-virtual one distinguished: {calls:?}" + ); + } +} diff --git a/crates/ruff_cpp_spo/src/lib.rs b/crates/ruff_cpp_spo/src/lib.rs index cbcdc708c29c36..17142a2a720b97 100644 --- a/crates/ruff_cpp_spo/src/lib.rs +++ b/crates/ruff_cpp_spo/src/lib.rs @@ -65,6 +65,12 @@ use ruff_spo_triplet::{ #[cfg(feature = "libclang")] mod clang_walker; +// The ordered behavioral ore — a SECOND walk that preserves the sequence, +// duplicates and scope structure the five-set body arm collapses. Additive: +// nothing in `clang_walker`'s output changes. See `events`'s module doc and +// `.claude/plans/behavioral-ore-v1.md`. +#[cfg(feature = "libclang")] +pub mod events; #[cfg(feature = "libclang")] pub use clang_walker::{ BodyArmConfig, MAPPED_CURSOR_KINDS, ParseDiagnostic, WalkError, class_body_cursor_histogram, From 5788e420aef1d251be3f6b5ec83ae2238335b8d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 22:12:23 +0000 Subject: [PATCH 11/19] Carry the overridden base method's IRI, not just a flag `virtually_overrides` is the one statement of behavioral relatedness the compiler makes for free, and a retrieval measure needs the PAIR. A boolean says a method overrides something without saying what, so any consumer comparing representations by distance had nothing to score against. `methods.tsv` gains a nineteenth column holding the base overload's IRI in the same per-overload form as the subject, so the two join directly. Also two clippy nits from the pedantic set: `map_or(false, ..)` becomes `is_some_and`, and the collapse test's `as u32` casts become `try_from`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/examples/harvest_events.rs | 5 +++-- crates/ruff_cpp_spo/src/events.rs | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/ruff_cpp_spo/examples/harvest_events.rs b/crates/ruff_cpp_spo/examples/harvest_events.rs index fa6df61f09b1a2..69dc5ee6dafb32 100644 --- a/crates/ruff_cpp_spo/examples/harvest_events.rs +++ b/crates/ruff_cpp_spo/examples/harvest_events.rs @@ -262,7 +262,7 @@ fn main() -> Result<(), Box> { } writeln!( me, - "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", cell(&m.iri), cell(&m.tu), cell(&m.class), @@ -280,7 +280,8 @@ fn main() -> Result<(), Box> { u8::from(m.overrides), m.mkind, m.access, - m.n_params + m.n_params, + m.overrides_target.as_deref().unwrap_or("-") )?; } let mut sy = String::new(); diff --git a/crates/ruff_cpp_spo/src/events.rs b/crates/ruff_cpp_spo/src/events.rs index 3d23c9b4960b82..bbbc85f9baf1dc 100644 --- a/crates/ruff_cpp_spo/src/events.rs +++ b/crates/ruff_cpp_spo/src/events.rs @@ -290,6 +290,12 @@ pub struct MethodOre { pub is_static: bool, pub is_virtual: bool, pub overrides: bool, + /// The base overload this method overrides, as an IRI — not just a flag. + /// It is the one compiler-given statement of behavioral relatedness the + /// corpus offers for free, and a retrieval measure needs the PAIR, so a + /// boolean would have left the geometry half of any downstream ablation + /// unmeasurable. + pub overrides_target: Option, pub mkind: &'static str, pub access: &'static str, pub n_params: u32, @@ -977,6 +983,10 @@ fn method_ore( is_static: m.is_static_method(), is_virtual: m.is_virtual_method(), overrides: m.get_overridden_methods().is_some_and(|o| !o.is_empty()), + overrides_target: m + .get_overridden_methods() + .and_then(|o| o.into_iter().next()) + .and_then(|b| method_iri(&b)), mkind: match m.get_kind() { EntityKind::Constructor => "ctor", EntityKind::Destructor => "dtor", From a0e9b29b3495c3a54d274644fb1a94b9e2f7a460 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 22:22:17 +0000 Subject: [PATCH 12/19] Allowlist libclang's own ParmDecl spelling The `typos` hook read `Parm` inside `EntityKind::ParmDecl` as a misspelling of `Param`. The identifier is libclang's own cursor-kind name, surfaced by the `clang` crate, so "correcting" it does not compile. Allowlisted as a whole identifier rather than as the bare word `Parm`, so a genuine `Parm`-for-`Param` typo elsewhere is still caught. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- _typos.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/_typos.toml b/_typos.toml index ff07e99b27e202..33ffa885a47c75 100644 --- a/_typos.toml +++ b/_typos.toml @@ -56,3 +56,6 @@ extend-ignore-re = [ [default.extend-identifiers] "FrIeNdLy" = "FrIeNdLy" +# libclang's own cursor-kind spelling, surfaced by the `clang` crate as +# `EntityKind::ParmDecl`. Renaming it does not compile. +"ParmDecl" = "ParmDecl" From 490b0d2269a37727fcca5266e47ce9d38bf0b1ea Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 22:35:30 +0000 Subject: [PATCH 13/19] Record the measured size of the sort+dedup defect The plan carried the defect and the control-fixture figure. On the 227-TU ladybug corpus the number is much worse: the shipped set arm retains 2.7% of ordered events, no adjacency and no scope structure, where the fixture suggested 11.2%. The gap is call sites. Also records what the loss costs in bits, and the decomposition that corrects the earlier framing: order and alphabet each buy roughly half, so restoring order alone would not have closed it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/plans/behavioral-ore-v1.md | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.claude/plans/behavioral-ore-v1.md b/.claude/plans/behavioral-ore-v1.md index c573456e81f609..eb48086bf00020 100644 --- a/.claude/plans/behavioral-ore-v1.md +++ b/.claude/plans/behavioral-ore-v1.md @@ -94,3 +94,45 @@ It is not resolved by editing the expectation to match the code. - **No new predicate.** The closed vocabulary is untouched; the ore is TSV beside the manifest, not a triple. - The extractor is a second walk over the same cursor, in its own module. + +## 6. Measured — 227-TU ladybug corpus, 2026-09-07 + +The §1 defect, quantified on real code rather than the control fixture +(2,217 methods, 95,365 ordered events): + +| | | +|---|---| +| ordered events | 95,365 | +| facts the shipped set arm keeps | 2,574 | +| duplicate occurrences destroyed | 2,093 | +| scope boundaries destroyed | 19,902 | +| adjacency pairs destroyed | 93,148 | +| **retained** | **2.7% of ordered events, 0 adjacency, 0 scope** | + +The control fixture said 11.2%; at corpus scale it is 2.7%. The gap is call +sites — 22,748 of them, of which the configured-mutator rule keeps 171. + +**What the loss costs, in bits.** Charging every representation for describing +the same held-out events (`bits/ore-event`, uncovered events costed at +`log2|alphabet|`): + +| representation | bits/ore-event | +|---|---| +| shipped sets | 4.0987 | +| ordered ore | 2.7455 | +| ordered ore + behavioural BPE (256 merges) | 2.6411 | + +The upstream fix is 13× the tokenizer's contribution. It decomposes into two +roughly equal, slightly sub-additive halves — **order** (same five kinds, +duplicates and adjacency restored) +0.7776, and **alphabet** (the 13 kinds the +set arm has no representation for, order still destroyed) +0.7001. Neither +alone explains it, so "restoring order is the wound" is half the story. + +**Provenance (the §15 check).** Events are 58% libclang-answered overall, but +`ScopeEnter` / `ScopeExit` / `Condition` / `Branch` are **0%** — they are +walk-derived. Every high-consistency motif the tokenizer learns is built from +those, so the recurring structure is contributed by the ordered walk, not by +the compiler. + +Full write-up, ladder and falsifier verdicts live with the experiment, not in +this repo; what belongs here is the defect and its size. From c0df2cca3c2c301338cdbb11ed6aab5fa5951bbc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 22:42:26 +0000 Subject: [PATCH 14/19] Fix the macOS scope test and four review findings The range-for scope test panicked "missing RangeFor" on macOS while passing on Linux. Its fixture included and ranged over a std::vector; macOS CI runs libclang without a sysroot, so the include failed, the parameter got an error type and the range-for never parsed. The other three loops in the same fixture need no header, which is why only RangeFor went missing. The fixture now ranges over a built-in array and needs no standard library at all. Reproduced locally by parsing both fixtures with -nostdinc++: the old one yields For/Function, the new one yields For/Function/RangeFor. Four findings from review, each verified against current code first: - A delete-expression's own type is void, so every Delete event was labelled void. The deleted type comes from the operand with one pointer layer removed. New is unaffected, since its own type is the allocated pointer type. - The compilation-database reader only understood "command". An entry using "arguments" left its file pending and the next entry's command was attributed to it, so that file was dropped and another could be given the wrong flags. Entries are now parsed independently and flushed at their closing brace, both command forms are read, and an entry carrying neither is reported. - The overridden-base IRI reached methods.tsv without the tab and newline guard every other cell goes through. - The "has a body" metric counted parameter events, so a method with parameters and an empty body was reported as having one. Also moved two test fixture writes inside the libclang lock. Twelve tests share the shapes fixture path and File::create truncates, so a write outside the lock could empty the file while another thread was parsing it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .../ruff_cpp_spo/examples/harvest_events.rs | 95 +++++++++++++++++-- crates/ruff_cpp_spo/src/clang_walker.rs | 11 ++- crates/ruff_cpp_spo/src/events.rs | 41 +++++--- 3 files changed, 123 insertions(+), 24 deletions(-) diff --git a/crates/ruff_cpp_spo/examples/harvest_events.rs b/crates/ruff_cpp_spo/examples/harvest_events.rs index 69dc5ee6dafb32..4ff3e10480ac4d 100644 --- a/crates/ruff_cpp_spo/examples/harvest_events.rs +++ b/crates/ruff_cpp_spo/examples/harvest_events.rs @@ -31,7 +31,7 @@ use std::collections::BTreeMap; use std::fmt::Write as _; use std::path::PathBuf; -use ruff_cpp_spo::events::{MethodOre, Symbols, walk_tu_events}; +use ruff_cpp_spo::events::{EventKind, MethodOre, Symbols, walk_tu_events}; /// A TSV cell can carry no tab and no newline. C++ identifiers and type /// spellings contain neither, but a malformed one would silently shift every @@ -44,7 +44,13 @@ fn cell(s: &str) -> String { /// flags that change what the parser SEES are kept — `-o`, `-c` and the input /// file itself would make libclang re-drive a compilation. fn args_of(cmd: &str) -> Vec { - let toks: Vec = shell_split(cmd); + keep_parse_flags(&shell_split(cmd)) +} + +/// The filter half of [`args_of`], over tokens that are ALREADY split. The +/// `arguments` form of a compilation database is a real array, so re-joining it +/// into a string just to re-split it could only lose information. +fn keep_parse_flags(toks: &[String]) -> Vec { let mut out = Vec::new(); let mut i = 0; while i < toks.len() { @@ -100,23 +106,77 @@ fn shell_split(s: &str) -> Vec { } /// `compile_commands.json` without a JSON dependency: the file is a flat array -/// of objects with string values, and only two keys are needed. +/// of objects, conventionally pretty-printed one field per line, which is the +/// shape this reads. +/// +/// Each object is parsed INDEPENDENTLY and flushed at its closing brace. An +/// earlier version kept `file` and `command` across objects and emitted as soon +/// as both were set; an entry using `arguments` therefore left its `file` +/// pending, and the NEXT object's `command` was attributed to the wrong file. +/// +/// Both command forms defined by the JSON compilation database are read: +/// `"command"` (one shell-escaped string) and `"arguments"` (an array of +/// already-split strings, which the spec prefers precisely because it needs no +/// shell unescaping). An entry carrying neither is reported rather than +/// silently dropped. fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { let mut out = Vec::new(); - let (mut file, mut cmd) = (None::, None::); + let mut file: Option = None; + let mut cmd: Option = None; + let mut argv: Option> = None; + let mut collecting: Option> = None; + let mut malformed = 0usize; + for raw in text.lines() { let line = raw.trim(); + + // Inside an `"arguments": [ ... ]` array, which may span lines. + if let Some(acc) = collecting.as_mut() { + for tok in line.split(',') { + if let Some(v) = json_string_field(tok.trim(), "") { + acc.push(v); + } + } + if line.contains(']') { + argv = collecting.take(); + } + continue; + } + if let Some(v) = json_string_field(line, "\"file\":") { file = Some(v); } else if let Some(v) = json_string_field(line, "\"command\":") { cmd = Some(v); + } else if let Some(rest) = line.strip_prefix("\"arguments\":") { + let rest = rest.trim_start().trim_start_matches('['); + let mut acc = Vec::new(); + for tok in rest.split(',') { + if let Some(v) = json_string_field(tok.trim(), "") { + acc.push(v); + } + } + if rest.contains(']') { + argv = Some(acc); + } else { + collecting = Some(acc); + } } - if let (Some(f), Some(c)) = (&file, &cmd) { - out.push((PathBuf::from(f), args_of(c))); - file = None; - cmd = None; + + // One object ended. Emit it, or count it, but never carry it forward. + if line.starts_with('}') { + match (file.take(), argv.take(), cmd.take()) { + // `arguments` wins: it is already split, so it cannot be + // mangled by re-splitting a shell-escaped string. + (Some(f), Some(a), _) => out.push((PathBuf::from(f), keep_parse_flags(&a))), + (Some(f), None, Some(c)) => out.push((PathBuf::from(f), args_of(&c))), + (Some(_), None, None) => malformed += 1, + (None, _, _) => {} + } } } + if malformed > 0 { + eprintln!("[ore] {malformed} compile_commands entries had neither command nor arguments"); + } out } @@ -281,7 +341,9 @@ fn main() -> Result<(), Box> { m.mkind, m.access, m.n_params, - m.overrides_target.as_deref().unwrap_or("-") + m.overrides_target + .as_deref() + .map_or_else(|| "-".to_string(), cell) )?; } let mut sy = String::new(); @@ -313,7 +375,20 @@ fn main() -> Result<(), Box> { syms.len(), out_dir.display() ); - let with_events = methods.iter().filter(|m| m.events.len() > 2).count(); + // A method scope emits ScopeEnter + ScopeExit, and each parameter emits one + // Param event, so `void f(int a) {}` reaches three events with an empty + // body. Only a non-prologue event counts as body content. + let with_events = methods + .iter() + .filter(|m| { + m.events.iter().any(|e| { + !matches!( + e.kind, + EventKind::ScopeEnter | EventKind::ScopeExit | EventKind::Param + ) + }) + }) + .count(); let a0_facts: u32 = methods .iter() .map(|m| m.set_reads + m.set_writes + m.set_raises + m.set_calls) diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index b09d042e1ea781..e4befacda0138b 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -1422,13 +1422,18 @@ mod arm_tests { let dir = std::env::temp_dir().join(format!("cpp_arm_{name}")); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); - let mut fh = std::fs::File::create(&path).expect("fixture file"); - fh.write_all(src.as_bytes()).expect("write fixture"); - drop(fh); + // The write happens INSIDE the lock. Twelve tests share the `shapes` + // fixture, so they share this path; `File::create` truncates, so a + // write outside the lock can empty the file while another thread is + // inside `parse()` on it. That parse then finds no methods and the + // caller's index panics. let _guard = CLANG_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut fh = std::fs::File::create(&path).expect("fixture file"); + fh.write_all(src.as_bytes()).expect("write fixture"); + drop(fh); let clang = Clang::new().expect("libclang"); let index = Index::new(&clang, false, false); let tu = index diff --git a/crates/ruff_cpp_spo/src/events.rs b/crates/ruff_cpp_spo/src/events.rs index bbbc85f9baf1dc..3d0671bd8cb6c5 100644 --- a/crates/ruff_cpp_spo/src/events.rs +++ b/crates/ruff_cpp_spo/src/events.rs @@ -663,11 +663,23 @@ impl<'s> Walk<'s> { EntityKind::NewExpr | EntityKind::DeleteExpr => { let is_new = e.get_kind() == EntityKind::NewExpr; - let ty = e - .get_type() - .map(|t| bare_type_name(&t.get_display_name())) - .filter(|t| !t.is_empty()) - .map(|t| self.syms.intern(SymKind::Type, Role::None, &t, Prov::Clang)); + // A delete-expression's OWN type is `void` -- it yields no + // value -- so reading it here labelled every Delete event + // `void`. The deleted object's type is the operand's, with one + // pointer layer removed: `delete p` where `p` is `Foo *` is a + // Delete of `Foo`. `new` is unaffected; its own type IS the + // allocated pointer type. + let ty = if is_new { + e.get_type().map(|t| bare_type_name(&t.get_display_name())) + } else { + e.get_children() + .first() + .and_then(Entity::get_type) + .map(|t| t.get_pointee_type().unwrap_or(t)) + .map(|t| bare_type_name(&t.get_display_name())) + } + .filter(|t| !t.is_empty()) + .map(|t| self.syms.intern(SymKind::Type, Role::None, &t, Prov::Clang)); let i = self.push( if is_new { EventKind::New @@ -1072,12 +1084,15 @@ mod tests { let dir = std::env::temp_dir().join(format!("cpp_ore_{name}")); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); - let mut fh = std::fs::File::create(&path).expect("fixture file"); - fh.write_all(src.as_bytes()).expect("write fixture"); - drop(fh); + // Written inside the lock. Every fixture here has a unique name today, + // so the paths do not collide, but `File::create` truncates and the + // sibling `arms_with` helper hit exactly that race on a shared path. let _guard = CLANG_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut fh = std::fs::File::create(&path).expect("fixture file"); + fh.write_all(src.as_bytes()).expect("write fixture"); + drop(fh); let mut syms = Symbols::default(); let tu = walk_tu_events(&path, &["-std=c++17".to_string()], &mut syms).expect("walk"); (tu.methods, syms) @@ -1206,17 +1221,21 @@ void C::b(int n) { if (n > 0) x_ = n; else x_ = 0; }", fn every_scope_kind_appears_with_the_right_parent_and_depth() { let (ms, _) = ore( "scopes", + // A built-in array, not `std::vector`, so the range-for needs no + // standard library. macOS CI runs libclang without a sysroot, so + // `#include ` fails there, `xs` gets an error type, and the + // range-for never parses -- the other three loops are header-free + // and passed, which is why only `RangeFor` went missing. r" -#include struct C { - void loops(const std::vector& xs, int n); + void loops(const int (&xs)[4], int n); void branches(int n); void protect(); void lam(int n); void blk(int n); int x_; }; -void C::loops(const std::vector& xs, int n) { +void C::loops(const int (&xs)[4], int n) { for (int i = 0; i < n; ++i) { x_ += i; } while (x_ < n) { ++x_; } do { --x_; } while (x_ > n); From f9a1b5072a6b9d4e8eb4b626b7cc1949e6ff003e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:26:03 +0000 Subject: [PATCH 15/19] Re-measure the sort+dedup defect on the whole corpus The recorded figures came from 227 of 1,081 translation units. The full harvest supersedes them, and the defect is worse at scale rather than better: the set arm retains 1.9% of ordered events, not 2.7%, because call sites dominate and the configured-mutator rule keeps 745 of 109,603. Adds the measurement the sample could not make. Override-pair retrieval was untestable at 227 units, where only 5 of 291 declared override targets had both sides present. At 1,081 units there are 446 usable pairs, and the set arm retrieves none of them while the ordered ore retrieves 26. The order-versus-alphabet decomposition and the provenance split both hold at 6.8 times the data. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .claude/plans/behavioral-ore-v1.md | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.claude/plans/behavioral-ore-v1.md b/.claude/plans/behavioral-ore-v1.md index eb48086bf00020..ee93fa104850cd 100644 --- a/.claude/plans/behavioral-ore-v1.md +++ b/.claude/plans/behavioral-ore-v1.md @@ -136,3 +136,43 @@ the compiler. Full write-up, ladder and falsifier verdicts live with the experiment, not in this repo; what belongs here is the defect and its size. + +## 7. Re-measured on the whole corpus — 1,081 TUs, 2026-09-07 + +§6 was the 227-TU sample. The full harvest supersedes its figures: **14,106 +methods, 646,411 ordered events, 36,224 symbols.** + +The defect gets WORSE with scale, because call sites dominate and the +configured-mutator rule keeps almost none of them: + +| | 227 TUs | 1,081 TUs | +|---|---|---| +| ordered events | 95,365 | 646,411 | +| facts the set arm keeps | 2,574 | 12,062 | +| call sites (kept) | 22,748 (171) | 109,603 (745) | +| adjacency pairs destroyed | 93,148 | 632,305 | +| **retained** | 2.7% | **1.9%** | + +The bits cost, same common denominator: + +| representation | 227 TUs | 1,081 TUs | +|---|---|---| +| shipped sets | 4.0987 | 4.2684 | +| ordered ore | 2.7455 | 2.7181 | +| + BPE (256 merges) | 2.6411 | 2.4721 | + +The two-halves decomposition holds at 6.8× the data — order +0.8254, alphabet ++0.6651, together +1.5503 — so neither alone explains the gap, and the upstream +fix remains several times the tokenizer's contribution. + +**Provenance, re-checked:** 54.1% of events are libclang-answered overall, and +`ScopeEnter` / `ScopeExit` / `Branch` / `Condition` / `Return` / `Cast` are still +exactly **0%**. The recurring structure is contributed by the ordered walk. + +**One measurement the sample got wrong.** At 227 TUs the ordered ore's +override-pair retrieval looked like a tie among representations, because only 5 +of 291 declared override targets had both sides present. At 1,081 TUs there are +**446** usable pairs, and the shipped set arm retrieves **0 of 446** while the +ordered ore retrieves 26 — a 5.3σ separation. A representation that cannot find +a method's override partner even once is the sharpest single statement of what +sort+dedup costs. From 5a9ea472e53650b545514f6da1cd39ac12e99fbc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:46:28 +0000 Subject: [PATCH 16/19] Fix five harvester defects found in review, each with a disable-verified test Five findings, each verified against current code first and each paired with a test confirmed to fail when the fix is removed: - The compilation-database reader split the "arguments" array on commas before parsing its strings, so a valid flag like -DPAIR=std::pair became two invalid fragments and both were dropped, silently changing the flags a translation unit was parsed with. The array is now scanned for quoted runs. - A standalone -I lost its directory. clang accepts both -Idir and -I dir, and keeping only the bare flag hands libclang an incomplete option and loses the include path, which shows up as a partial AST rather than an error. - The override target omitted the ref qualifier while the method IRI includes it, so Base::f() & and Base::f() && both pointed at Base.f() and neither joined to the base method it actually overrides. - A subscripted assignment recorded its array as a read as well as a write. Skipping the target needed the same descent the target lookup uses, because the base arrives wrapped and a bare member-reference test never matches it. - A dereference assignment recorded the pointer member as written. *ptr_ = v changes the pointee; the member still holds the same address. A sixth finding, that a parameter's default argument contributes body facts, is NOT reproducible. Measured: with the parameter skip removed, a default argument containing both a static call and a static member read contributed no calls and no reads, because the call arm only records specific receiver shapes and reads cover own non-static members. The skip stays as scoping, its comment records the measurement, and no test claims it fixes an observed defect. The example is now a test target, so its parser tests actually run. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/Cargo.toml | 4 + .../ruff_cpp_spo/examples/harvest_events.rs | 145 +++++++++++-- crates/ruff_cpp_spo/src/clang_walker.rs | 200 +++++++++++++++++- 3 files changed, 323 insertions(+), 26 deletions(-) diff --git a/crates/ruff_cpp_spo/Cargo.toml b/crates/ruff_cpp_spo/Cargo.toml index d68c37f77716d7..8b4c4d08f52e02 100644 --- a/crates/ruff_cpp_spo/Cargo.toml +++ b/crates/ruff_cpp_spo/Cargo.toml @@ -69,3 +69,7 @@ workspace = true [[example]] name = "harvest_events" required-features = ["libclang"] +# The compilation-database reader lives here and has its own unit tests, so the +# example is a test target too. Without this, `cargo test` builds it but never +# runs those tests, and a parser regression ships silently. +test = true diff --git a/crates/ruff_cpp_spo/examples/harvest_events.rs b/crates/ruff_cpp_spo/examples/harvest_events.rs index 4ff3e10480ac4d..b1ea5eabb40fb3 100644 --- a/crates/ruff_cpp_spo/examples/harvest_events.rs +++ b/crates/ruff_cpp_spo/examples/harvest_events.rs @@ -55,7 +55,10 @@ fn keep_parse_flags(toks: &[String]) -> Vec { let mut i = 0; while i < toks.len() { let t = &toks[i]; - if t == "-isystem" || t == "-include" { + // `-I` can be written joined (`-Iinc`) or as two tokens (`-I inc`); + // clang accepts both. Keeping only the bare `-I` would hand libclang an + // incomplete flag and lose the include path. + if matches!(t.as_str(), "-I" | "-isystem" | "-include") { if let Some(v) = toks.get(i + 1) { out.push(t.clone()); out.push(v.clone()); @@ -124,21 +127,21 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { let mut file: Option = None; let mut cmd: Option = None; let mut argv: Option> = None; - let mut collecting: Option> = None; + let mut collecting: Option = None; let mut malformed = 0usize; for raw in text.lines() { let line = raw.trim(); - // Inside an `"arguments": [ ... ]` array, which may span lines. + // Inside an `"arguments": [ ... ]` array, which may span lines. The + // raw text is accumulated and scanned for quoted runs at the closing + // bracket, NEVER split on commas: `-DPAIR=std::pair` is one + // valid argument, and splitting it would silently drop the flag. if let Some(acc) = collecting.as_mut() { - for tok in line.split(',') { - if let Some(v) = json_string_field(tok.trim(), "") { - acc.push(v); - } - } + acc.push_str(line); if line.contains(']') { - argv = collecting.take(); + argv = Some(json_string_array(acc)); + collecting = None; } continue; } @@ -148,17 +151,11 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { } else if let Some(v) = json_string_field(line, "\"command\":") { cmd = Some(v); } else if let Some(rest) = line.strip_prefix("\"arguments\":") { - let rest = rest.trim_start().trim_start_matches('['); - let mut acc = Vec::new(); - for tok in rest.split(',') { - if let Some(v) = json_string_field(tok.trim(), "") { - acc.push(v); - } - } + let rest = rest.trim_start(); if rest.contains(']') { - argv = Some(acc); + argv = Some(json_string_array(rest)); } else { - collecting = Some(acc); + collecting = Some(rest.to_string()); } } @@ -180,6 +177,43 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { out } +/// Every quoted run in a JSON array of strings, in order. +/// +/// Scans for unescaped `"` delimiters rather than splitting on `,`, because a +/// comma inside a string is CONTENT: `["-DPAIR=std::pair"]` is one +/// argument, and a comma split turns it into two invalid fragments that are +/// then dropped, silently changing the flags a translation unit is parsed with. +fn json_string_array(text: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let (mut inside, mut esc) = (false, false); + for c in text.chars() { + if !inside { + if c == '"' { + inside = true; + cur.clear(); + } + continue; + } + if esc { + cur.push(match c { + 'n' => '\n', + 't' => '\t', + other => other, + }); + esc = false; + } else if c == '\\' { + esc = true; + } else if c == '"' { + inside = false; + out.push(std::mem::take(&mut cur)); + } else { + cur.push(c); + } + } + out +} + fn json_string_field(line: &str, key: &str) -> Option { let rest = line.strip_prefix(key)?.trim_start(); let body = rest.strip_prefix('"')?; @@ -405,3 +439,78 @@ fn main() -> Result<(), Box> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::{json_string_array, keep_parse_flags, parse_cc_json}; + + /// A comma INSIDE a JSON string is content, not a separator. Splitting on + /// commas turns one valid flag into two invalid fragments and drops both, + /// silently changing the flags a translation unit is parsed with. + #[test] + fn an_argument_containing_a_comma_survives_the_array_parse() { + let got = json_string_array(r#"["c++", "-DPAIR=std::pair", "-c"]"#); + assert_eq!(got, ["c++", "-DPAIR=std::pair", "-c"]); + // The whole point: the comma-bearing flag is ONE element, not two. + assert_eq!(got.len(), 3, "a comma inside a string must not split it"); + } + + /// clang accepts `-I dir` as two tokens. Keeping the bare `-I` and dropping + /// the directory hands libclang an incomplete flag and loses the include + /// path, which shows up as a partial AST rather than an error. + #[test] + fn a_standalone_include_flag_keeps_its_directory() { + let toks: Vec = ["-I", "/opt/incdir", "-c", "a.cpp", "-Ijoined"] + .iter() + .map(|s| (*s).to_string()) + .collect(); + assert_eq!(keep_parse_flags(&toks), ["-I", "/opt/incdir", "-Ijoined"]); + } + + /// Both command forms are read, each entry is flushed at its own closing + /// brace, and an entry carrying neither form never steals a later command. + #[test] + fn each_compilation_database_entry_is_parsed_independently() { + let db = r#"[ +{ + "directory": "/tmp", + "arguments": ["c++", "-std=c++17", "-DPAIR=std::pair", "-I", "/opt/incdir", "-c", "a.cpp"], + "file": "/tmp/a.cpp" +}, +{ + "directory": "/tmp", + "file": "/tmp/no_command.cpp" +}, +{ + "directory": "/tmp", + "command": "c++ -std=c++17 -I/other -c b.cpp", + "file": "/tmp/b.cpp" +} +]"#; + let got = parse_cc_json(db); + let files: Vec = got.iter().map(|(f, _)| f.display().to_string()).collect(); + // The entry with neither form is dropped, and crucially it does NOT + // pair with the next entry's command. + assert_eq!(files, ["/tmp/a.cpp", "/tmp/b.cpp"]); + assert_eq!( + got[0].1, + [ + "-std=c++17", + "-DPAIR=std::pair", + "-I", + "/opt/incdir" + ] + ); + assert_eq!(got[1].1, ["-std=c++17", "-I/other"]); + } + + /// A multiline `arguments` array is accumulated before it is scanned, so a + /// comma-bearing flag survives that layout too. + #[test] + fn a_multiline_arguments_array_is_parsed_as_one_unit() { + let db = "[\n{\n \"file\": \"/tmp/c.cpp\",\n \"arguments\": [\n \"c++\",\n \"-DPAIR=std::pair\",\n \"-Iinc\"\n ]\n}\n]"; + let got = parse_cc_json(db); + assert_eq!(got.len(), 1); + assert_eq!(got[0].1, ["-DPAIR=std::pair", "-Iinc"]); + } +} diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index e4befacda0138b..004afaa03e76eb 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -802,8 +802,20 @@ fn build_method(m: &Entity, arm: Option<&BodyArmConfig>) -> CppMethod { .flatten() .filter_map(|a| a.get_type().map(|t| t.get_display_name())) .collect(); + // The ref qualifier is part of method identity, so it has to be + // part of the override TARGET too. Without it `Base::f() &` and + // `Base::f() &&` both point at `Base.f()`, and neither joins to + // the base node the IRI actually names. + let refq = base_m + .get_type() + .and_then(|t| t.get_ref_qualifier()) + .map(|q| match q { + RefQualifier::LValue => " &", + RefQualifier::RValue => " &&", + }) + .unwrap_or(""); Some(format!( - "{}.{mname}({}){}", + "{}.{mname}({}){}{refq}", qualified_name(&parent), params.join(","), if base_m.is_const_method() { @@ -1010,7 +1022,23 @@ enum GuardedBranch { #[cfg(feature = "libclang")] pub(crate) fn method_body_arm(method: &Entity, cfg: &BodyArmConfig) -> BodyArm { let mut arm = BodyArm::default(); - walk_body(method, &mut arm, cfg, None); + // Parameters are skipped rather than selecting the CompoundStmt, because a + // constructor's member-initialiser list is a SIBLING of the body, not a + // child of it — narrowing to the body would silently drop ctor-init facts. + // + // This is SCOPING, not a bug fix, and the difference is measured: with this + // skip removed, `void put(int v = C::fallback() + C::s_)` still contributed + // no calls and no reads, because the call arm only records specific + // receiver shapes and reads only cover own non-static members. A default + // argument is not something the body executes, so it has no business being + // walked — but no leak is reachable through it today, and a test asserting + // otherwise would pass whether or not this line is here. + for child in method.get_children() { + if child.get_kind() == EntityKind::ParmDecl { + continue; + } + walk_node(&child, &mut arm, cfg, None); + } for facts in [ &mut arm.writes, &mut arm.reads, @@ -1033,6 +1061,70 @@ fn walk_body(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Optio } } +/// The LHS of an assignment, minus the member reference that IS the target. +/// +/// `arr_[i] = v` must record the subscript `i` as a read and `arr_` only as a +/// write; the naive child walk records `arr_` twice, once through each role. +#[cfg(feature = "libclang")] +fn walk_lhs_skipping_target( + lhs: &Entity, + arm: &mut BodyArm, + cfg: &BodyArmConfig, + guard: Option<&str>, +) { + let target = assignment_target(lhs); + for child in lhs.get_children() { + // Compared through `assignment_target`, not `own_member_name`: the + // subscript's base arrives wrapped in an UnexposedExpr, so a bare + // member-reference test never matches it and the array is re-read. + if target.is_some() && assignment_target(&child) == target { + walk_past_target(&child, arm, cfg, guard); + continue; + } + walk_node(&child, arm, cfg, guard); + } +} + +/// Walk everything under the write target EXCEPT the target's own reference. +/// +/// Descending with `walk_body` is not enough: the reference arrives wrapped in +/// an `UnexposedExpr`, so walking the wrapper's children lands straight back on +/// the `MemberRefExpr` and records the read this exists to avoid. Follow the +/// same descent `assignment_target` uses, and walk only what QUALIFIES the +/// member (`this`, or `p` in `p.x_`) plus any subscript indices passed on the +/// way down. +#[cfg(feature = "libclang")] +fn walk_past_target(e: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { + let mut cur = *e; + for _ in 0..32 { + match cur.get_kind() { + EntityKind::MemberRefExpr => { + walk_body(&cur, arm, cfg, guard); + return; + } + EntityKind::UnexposedExpr | EntityKind::ParenExpr => { + let Some(next) = cur.get_children().into_iter().next() else { + return; + }; + cur = next; + } + // A nested subscript (`a_[i][j]`): the base continues the descent, + // and every index is a real read that must not be lost with it. + EntityKind::ArraySubscriptExpr => { + let mut children = cur.get_children().into_iter(); + let Some(base) = children.next() else { + return; + }; + for idx in children { + walk_node(&idx, arm, cfg, guard); + } + cur = base; + } + _ => return, + } + } +} + /// Walk ONE node, matching it before descending. #[cfg(feature = "libclang")] fn walk_node(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Option<&str>) { @@ -1054,8 +1146,11 @@ fn walk_node(node: &Entity, arm: &mut BodyArm, cfg: &BodyArmConfig, guard: Optio record_write(arm, lhs, guard); // The LHS's own member reference is the write TARGET, not a // read, so descend past it — but keep whatever it contains - // (`arr_[i]`'s subscript, `p` in `p.x_`). - walk_body(lhs, arm, cfg, guard); + // (`arr_[i]`'s subscript, `p` in `p.x_`). Walking the children + // blindly re-reads the target through the nested member + // reference, which is how `arr_[i] = v` recorded `arr_` as + // both a write and a read. + walk_lhs_skipping_target(lhs, arm, cfg, guard); for r in rest { walk_node(r, arm, cfg, guard); } @@ -1210,12 +1305,13 @@ pub(crate) fn assignment_target(lhs: &Entity) -> Option { for _ in 0..32 { match cur.get_kind() { EntityKind::MemberRefExpr => return own_member_name(&cur), - EntityKind::UnexposedExpr - | EntityKind::ParenExpr - | EntityKind::ArraySubscriptExpr - | EntityKind::UnaryOperator => { + EntityKind::UnexposedExpr | EntityKind::ParenExpr | EntityKind::ArraySubscriptExpr => { cur = cur.get_children().into_iter().next()?; } + // NOT UnaryOperator. `*ptr_ = v` changes the POINTEE; the member + // `ptr_` itself is unchanged, so recording it as a write is wrong. + // Increment/decrement reaches this function with the operand + // already unwrapped, so it is unaffected. _ => return None, } } @@ -1496,6 +1592,94 @@ struct Patient { ); } + /// `arr_[idx_] = v` writes `arr_` and reads `idx_`. Walking the LHS's + /// children blindly also re-read `arr_` through the nested member + /// reference, so one assignment reported the array as both written and + /// read — and a read the method never performs changes its recipe. + #[test] + fn a_subscripted_assignment_reads_the_index_but_not_the_array() { + let a = &arms( + "subscript", + r" +struct C { + void put(int v) { arr_[idx_] = v; } + int arr_[8]; + int idx_; +};", + )["put"]; + assert_eq!(a.writes, ["arr_"]); + assert_eq!(a.reads, ["idx_"], "the write target must not be a read too"); + } + + /// `*ptr_ = v` changes the POINTEE. The member `ptr_` still holds the same + /// address afterwards, so recording it as a write claims a mutation that + /// did not happen. + #[test] + fn a_dereference_assignment_does_not_write_the_pointer_member() { + let a = &arms( + "deref", + r" +struct C { + void put(int v) { *ptr_ = v; } + int* ptr_; +};", + )["put"]; + assert!( + !a.writes.iter().any(|w| w == "ptr_"), + "the pointer member is unchanged: {:?}", + a.writes + ); + } + + /// The override TARGET must carry the ref qualifier, because the method + /// IRI does. Without it `Base::f() &` and `Base::f() &&` both point at + /// `Base.f()`, so neither joins to the base node it actually overrides — + /// and that pair is the only compiler-given statement of behavioural + /// relatedness the corpus offers for free. + #[test] + fn an_override_target_distinguishes_the_ref_qualified_overloads() { + let src = r" +struct Base { + virtual void f() & ; + virtual void f() && ; +}; +struct D : Base { + void f() & override {} + void f() && override {} +}; +"; + let dir = std::env::temp_dir().join("cpp_refq_override"); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("f.cpp"); + let _guard = CLANG_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::fs::write(&path, src).expect("write fixture"); + let (classes, _) = walk_tu_configured( + &path, + &["-std=c++17".to_string()], + Some(&BodyArmConfig::default()), + ) + .expect("walk"); + let d = classes.iter().find(|c| c.name == "D").expect("class D"); + let targets: Vec<&str> = d + .declarations + .iter() + .filter_map(|decl| match decl { + Declaration::Method(m) if m.name == "f" => m.overrides.as_deref(), + _ => None, + }) + .collect(); + assert!( + targets.contains(&"Base.f() &"), + "the lvalue overload must name its own base overload: {targets:?}" + ); + assert!( + targets.contains(&"Base.f() &&"), + "the rvalue overload must name its own base overload: {targets:?}" + ); + } + /// The defect the token-based operator lookup exists to fix: without it /// every binary operator's left operand read as a write, so a pure /// comparison reported a mutation. From 00330895f27abad88f00617330869311afce3ba0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 23:50:43 +0000 Subject: [PATCH 17/19] Give each test fixture its own path, and correct what the lock does cargo test (linux) failed on f9a1b50 with "no entry found for key" from the shared shapes fixture: the parse found no methods, so indexing the result panicked. That is the truncation mode reported in review, and my earlier fix for it does not work. Moving the write inside CLANG_TEST_LOCK cannot help. The lock is a plain Mutex in one process, and this repository runs tests under nextest, which gives every test its OWN process. Twelve tests shared cpp_arm_shapes/f.cpp, File::create truncates, so one process could empty the file while another was inside parse(). No amount of in-process locking reaches across that boundary. Every fixture path now carries a per-process, per-call salt, which removes the sharing rather than trying to coordinate it. The comments that credited the lock with fixing this are corrected; the lock still has a job, serialising libclang within a process, and only that. Honest limit: a harness running 240 concurrent single-test processes did not reproduce the failure either before or after this change, so it is not evidence the fix works. The argument is structural instead. The precondition was read directly from the code, and unique paths make the failure mode impossible by construction rather than merely unlikely. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/src/clang_walker.rs | 43 ++++++++++++++++++------- crates/ruff_cpp_spo/src/events.rs | 11 ++++--- crates/ruff_cpp_spo/src/lib.rs | 20 +++++++++--- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index 004afaa03e76eb..b8c7080c9f5064 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -1486,6 +1486,28 @@ pub(crate) fn bare_type_name(display: &str) -> String { #[cfg(all(test, feature = "libclang"))] pub(crate) static CLANG_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +/// A per-process, per-call suffix for a test fixture directory. +/// +/// [`CLANG_TEST_LOCK`] serialises libclang WITHIN a process, and that is all it +/// can do: this repository runs tests under nextest, which gives every `#[test]` +/// its OWN process, so a `Mutex` in one of them is invisible to the others. +/// Twelve tests once shared `cpp_arm_shapes/f.cpp`, and `File::create` +/// truncates, so one process could empty the file while another was inside +/// `parse()`. That parse then found no methods and the caller's index panicked +/// with `no entry found for key` — intermittently, which is the worst way to +/// find out. Unique paths remove the sharing instead of trying to coordinate +/// it. +#[cfg(test)] +pub(crate) fn fixture_salt() -> String { + use std::sync::atomic::{AtomicU32, Ordering}; + static N: AtomicU32 = AtomicU32::new(0); + format!( + "{}_{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + ) +} + #[cfg(all(test, feature = "libclang"))] mod arm_tests { use super::*; @@ -1515,15 +1537,14 @@ mod arm_tests { /// One parse per fixture, not one per method: `Clang` is a process /// singleton, so each parse serialises on [`CLANG_TEST_LOCK`]. fn arms_with(name: &str, src: &str, cfg: &BodyArmConfig) -> BTreeMap { - let dir = std::env::temp_dir().join(format!("cpp_arm_{name}")); + let dir = std::env::temp_dir().join(format!("cpp_arm_{name}_{}", fixture_salt())); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); - // The write happens INSIDE the lock. Twelve tests share the `shapes` - // fixture, so they share this path; `File::create` truncates, so a - // write outside the lock can empty the file while another thread is - // inside `parse()` on it. That parse then finds no methods and the - // caller's index panics. + // The write is inside the lock, which serialises libclang within THIS + // process. It does not make the fixture safe on its own: see + // `fixture_salt`, which is what actually stops two test PROCESSES + // sharing a path. let _guard = CLANG_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1648,7 +1669,7 @@ struct D : Base { void f() && override {} }; "; - let dir = std::env::temp_dir().join("cpp_refq_override"); + let dir = std::env::temp_dir().join(format!("cpp_refq_override_{}", fixture_salt())); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); let _guard = CLANG_TEST_LOCK @@ -1908,7 +1929,7 @@ struct Svc { }; void Svc::finish(int v) { status_ = v; repo_.Save(); } "#; - let dir = std::env::temp_dir().join("cpp_arm_outofline"); + let dir = std::env::temp_dir().join(format!("cpp_arm_outofline_{}", fixture_salt())); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); std::fs::write(&path, src).expect("write fixture"); @@ -1955,7 +1976,7 @@ struct Holder { void value() { plain_only_ = 3; } }; "#; - let dir = std::env::temp_dir().join("cpp_refqual"); + let dir = std::env::temp_dir().join(format!("cpp_refqual_{}", fixture_salt())); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); std::fs::write(&path, src).expect("write fixture"); @@ -2011,7 +2032,7 @@ struct Holder { let src = r#" struct Svc { int status_; void set(int v) { status_ = v; } }; "#; - let dir = std::env::temp_dir().join("cpp_arm_sigonly"); + let dir = std::env::temp_dir().join(format!("cpp_arm_sigonly_{}", fixture_salt())); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); std::fs::write(&path, src).expect("write fixture"); @@ -2062,7 +2083,7 @@ mod walker_tests { /// Write `src` to a fresh temp file under a name-scoped dir (mirrors /// `arm_tests::arm_of`'s fixture-writing pattern), returning its path. fn write_fixture(name: &str, src: &str) -> std::path::PathBuf { - let dir = std::env::temp_dir().join(format!("cpp_walker_{name}")); + let dir = std::env::temp_dir().join(format!("cpp_walker_{name}_{}", fixture_salt())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("f.cpp"); let mut fh = std::fs::File::create(&path).unwrap(); diff --git a/crates/ruff_cpp_spo/src/events.rs b/crates/ruff_cpp_spo/src/events.rs index 3d0671bd8cb6c5..0c38982ab20629 100644 --- a/crates/ruff_cpp_spo/src/events.rs +++ b/crates/ruff_cpp_spo/src/events.rs @@ -1081,12 +1081,15 @@ mod tests { /// here serialises on the crate-wide lock — the same lock `arm_tests` and /// `libclang_tests` take. fn ore(name: &str, src: &str) -> (Vec, Symbols) { - let dir = std::env::temp_dir().join(format!("cpp_ore_{name}")); + let dir = std::env::temp_dir().join(format!( + "cpp_ore_{name}_{}", + crate::clang_walker::fixture_salt() + )); std::fs::create_dir_all(&dir).expect("temp dir"); let path = dir.join("f.cpp"); - // Written inside the lock. Every fixture here has a unique name today, - // so the paths do not collide, but `File::create` truncates and the - // sibling `arms_with` helper hit exactly that race on a shared path. + // The path carries `fixture_salt` so two test PROCESSES never share it; + // the lock only serialises libclang within one process, which is all a + // `Mutex` can do when the runner gives every test its own process. let _guard = CLANG_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); diff --git a/crates/ruff_cpp_spo/src/lib.rs b/crates/ruff_cpp_spo/src/lib.rs index 17142a2a720b97..62e205f1b2f76b 100644 --- a/crates/ruff_cpp_spo/src/lib.rs +++ b/crates/ruff_cpp_spo/src/lib.rs @@ -1098,7 +1098,10 @@ class Recognizer : public Classify { let _guard = CLANG_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let dir = std::env::temp_dir().join("cpp_merge_body_arms"); + let dir = std::env::temp_dir().join(format!( + "cpp_merge_body_arms_{}", + crate::clang_walker::fixture_salt() + )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).expect("temp dir"); // `a_header.h` sorts before `b_source.cpp`, so the arm-less sighting @@ -1189,7 +1192,10 @@ class Recognizer : public Classify { let _guard = CLANG_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let dir = std::env::temp_dir().join("cpp_merge_refqual"); + let dir = std::env::temp_dir().join(format!( + "cpp_merge_refqual_{}", + crate::clang_walker::fixture_salt() + )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).expect("temp dir"); std::fs::write( @@ -1497,7 +1503,10 @@ class Recognizer : public Classify { let _guard = CLANG_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let base = std::env::temp_dir().join("ruff_cpp_spo_tree_fixture"); + let base = std::env::temp_dir().join(format!( + "ruff_cpp_spo_tree_fixture_{}", + crate::clang_walker::fixture_salt() + )); let sub = base.join("sub"); let _ = std::fs::remove_dir_all(&base); std::fs::create_dir_all(&sub).expect("mkdir tree"); @@ -1543,7 +1552,10 @@ class Recognizer : public Classify { let _guard = CLANG_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let base = std::env::temp_dir().join("ruff_cpp_spo_symlink_fixture"); + let base = std::env::temp_dir().join(format!( + "ruff_cpp_spo_symlink_fixture_{}", + crate::clang_walker::fixture_salt() + )); let sub = base.join("sub"); let _ = std::fs::remove_dir_all(&base); std::fs::create_dir_all(&sub).expect("mkdir"); From e8e611e07548cefa60c4f049d13289aa8970dd0a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 00:16:47 +0000 Subject: [PATCH 18/19] Require the member to be the subject of a guard condition, not merely in it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `absence_guard` classified any single-own-member condition by reading TOKENS across the whole condition, and never asked where in it the member sat. So a condition that merely MENTIONS the member was read as a test OF the member, and the write in the branch it picked became a `guarded_write` — which `recipe::classify` prefers over `Compute`/`Normalize`, so the method was classified by a guard its source does not contain. Measured before the fix, seven shapes were wrong: if (is_ready(ptr_)) else ptr_ = v guarded if (!is_ready(ptr_)) then ptr_ = v guarded if (is_ready(ptr_) == 0) then ptr_ = v guarded <- the comparison arm has the same hole, so this is not only the fallback if (count_ + 1) else count_ = v guarded if (!(count_ + 1)) then count_ = v guarded if (*ptr_) else ptr_ = v guarded <- tests the pointee if (!*ptr_) then ptr_ = v guarded `member_is_condition_subject` walks down from the condition through only those positions that keep the member the subject: wrappers and parentheses, a logical `!`, either side of a comparison, and the RECEIVER of a call. Not a call's arguments, not an operand of arithmetic, not a dereference. The token table is unchanged; this is a precondition in front of it. Two things measurement decided that guessing would have got wrong. An overloaded operator is also a `CallExpr`, and there the first child is the left OPERAND rather than a callee, so `sp_ + 1` on a smart pointer reached the member through the ordinary-call path and stayed a guard even after the builtin `+` was rejected. The arm now reads the callee's name and treats operators as operators. Restricting the receiver allowance to NULLARY calls looked tidy and is wrong. It costs three real guards on the corpus, among them `if (prevSelVector == nullptr)` — a plain null test that is only a call at all because the member is a smart pointer, so `==` is overloaded and arrives with operands. `head.valid()` is the same story for the receiver chain: a rule hardcoding `empty` would have dropped it. Corpus impact: none. The 26 translation units carrying the 33 guarded writes were re-harvested, and the same 33 methods come back — every guard the corpus actually records was already a genuine subject test, so the figures in `.claude/plans/behavioral-ore-v1.md` are unaffected. The defect was latent here, not active. That is also why the fixtures are the evidence the fix works and the corpus is only evidence it costs nothing. Each negative fixture writes the member in exactly the branch the pre-fix rule guarded, because the misclassification is silent when the write lands in the other branch — a fixture with the write on the wrong side passes without the fix. Three disable runs, each red then green: removing the precondition fails only the negative test; making it reject everything fails only the three positive tests; dropping the operator arm fails only the overloaded case. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- crates/ruff_cpp_spo/src/clang_walker.rs | 166 +++++++++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/ruff_cpp_spo/src/clang_walker.rs b/crates/ruff_cpp_spo/src/clang_walker.rs index b8c7080c9f5064..93619bad9ca254 100644 --- a/crates/ruff_cpp_spo/src/clang_walker.rs +++ b/crates/ruff_cpp_spo/src/clang_walker.rs @@ -1388,6 +1388,9 @@ fn absence_guard(cond: &Entity) -> Option<(String, GuardedBranch)> { let [member] = members.as_slice() else { return None; }; + if !member_is_condition_subject(cond, 64) { + return None; + } let tokens: Vec = cond .get_range()? @@ -1417,6 +1420,81 @@ fn absence_guard(cond: &Entity) -> Option<(String, GuardedBranch)> { Some((member.clone(), branch)) } +/// Is the condition's own-member reference the SUBJECT of the test, or does the +/// condition merely MENTION it? +/// +/// `if (ptr_)`, `if (!ptr_)`, `if (ptr_ == nullptr)` and `if (name_.empty())` +/// all test the member itself. `if (is_ready(ptr_))` and `if (count_ + 1)` do +/// not: there the member is an argument or an operand, and the predicate's +/// result says nothing about whether the member is absent. Before this check +/// both were classified as guards, because the branch table reads TOKENS +/// across the whole condition and never asked where in it the member sits — so +/// `ptr_ = make()` in the other branch was recorded as a schema default. +/// +/// The walk descends only through positions that keep the member the subject: +/// wrappers and parentheses, a logical `!`, either side of a comparison, and +/// the RECEIVER of a call. Never a call's arguments, never an operand of +/// arithmetic, and never a dereference — `if (*ptr_)` tests the pointee, and +/// the member holds the same address whichever way that goes. +#[cfg(feature = "libclang")] +fn member_is_condition_subject(node: &Entity, depth: u32) -> bool { + // Bounded so a cyclic cursor graph cannot loop; expression nesting in a + // condition is far shallower than this. + let Some(depth) = depth.checked_sub(1) else { + return false; + }; + let children = node.get_children(); + let any = |cs: &[Entity]| cs.iter().any(|c| member_is_condition_subject(c, depth)); + match node.get_kind() { + // Either this IS the member, or it is a receiver chain leading to it + // (`name_.empty()` reaches `name_` through the callee reference). + EntityKind::MemberRefExpr => own_member_name(node).is_some() || any(&children), + EntityKind::UnexposedExpr | EntityKind::ParenExpr => any(&children), + // `!x` keeps `x` the subject. `*p`, `-n`, `&x`, `++i` do not. + EntityKind::UnaryOperator => { + unary_operator_spelling(node).as_deref() == Some("!") && any(&children) + } + // A comparison tests its operands; arithmetic and logic do not. + EntityKind::BinaryOperator => { + matches!( + binary_operator_spelling(node).as_deref(), + Some("==" | "!=" | "<" | ">" | "<=" | ">=") + ) && any(&children) + } + // An OVERLOADED operator is also a `CallExpr`, and there the first + // child is the left OPERAND rather than a callee — so `sp_ + 1` on a + // smart pointer would reach `sp_` through the ordinary-call path and + // become a guard, which the builtin-`+` case above rejects. Read the + // callee's name and treat operators as operators: a comparison, a + // contextual `operator bool`, or an `operator!` keeps the member the + // subject; anything else (arithmetic, `[]`, `*`, `->`) does not. + // + // A name that merely BEGINS with `operator` but is not in that set is + // rejected rather than followed, which is the conservative direction. + EntityKind::CallExpr => match node.get_name().as_deref() { + Some( + "operator==" | "operator!=" | "operator<" | "operator>" | "operator<=" + | "operator>=" | "operator bool" | "operator!", + ) => any(&children), + Some(n) if n.starts_with("operator") => false, + // An ordinary call: only the callee/receiver chain, which IS the + // first child here. The arguments are deliberately not followed. + _ => children + .first() + .is_some_and(|c| member_is_condition_subject(c, depth)), + }, + _ => false, + } +} + +/// The operator spelling of a PREFIX unary operator, read off the token stream +/// — libclang exposes no unary-operator kind either. Postfix `i++` yields `i`, +/// which is what callers testing for `!` want. +#[cfg(feature = "libclang")] +fn unary_operator_spelling(node: &Entity) -> Option { + Some(node.get_range()?.tokenize().first()?.get_spelling()) +} + /// Every own data member referenced anywhere under `node`. #[cfg(feature = "libclang")] fn collect_own_members(node: &Entity, out: &mut Vec) { @@ -1792,16 +1870,27 @@ struct D : Base { const GUARDS: &str = r#" struct Str { bool empty() const; }; +struct Sp { bool operator==(decltype(nullptr)) const; explicit operator bool() const; }; struct Cfg { int* ptr_; Str name_; int other_; + int count_; + Sp sp_; void null_guard(int* v) { if (ptr_ == nullptr) { ptr_ = v; } } void bang_guard(int* v) { if (!ptr_) { ptr_ = v; } } void empty_guard(Str s) { if (name_.empty()) { name_ = s; } } void present_guard(int* v) { if (ptr_ != nullptr) { } else { ptr_ = v; } } void bare_truth_guard(int* v) { if (ptr_) { } else { ptr_ = v; } } void not_empty_guard(Str s) { if (!name_.empty()) { } else { name_ = s; } } + void paren_truth_guard(int* v) { if ((ptr_)) { } else { ptr_ = v; } } + void this_truth_guard(int* v) { if (this->ptr_) { } else { ptr_ = v; } } + void zero_cmp_guard(int v) { if (count_ == 0) { count_ = v; } } + // A smart-pointer-shaped member: `==` and the truth test are OVERLOADED + // operators, so both arrive as calls. `prevSelVector == nullptr` in the + // real corpus is exactly this shape. + void sp_null_guard(Sp v) { if (sp_ == nullptr) { sp_ = v; } } + void sp_truth_guard(Sp v) { if (sp_) { } else { sp_ = v; } } void unguarded(int* v) { ptr_ = v; } void wrong_branch(int* v) { if (ptr_ == nullptr) { } else { ptr_ = v; } } void other_member_guard(int* v) { if (other_ == 0) { ptr_ = v; } } @@ -1814,7 +1903,13 @@ struct Cfg { #[test] fn an_absence_test_makes_the_then_branch_write_a_guarded_write() { let all = arms("guards", GUARDS); - for name in ["null_guard", "bang_guard", "empty_guard"] { + for name in [ + "null_guard", + "bang_guard", + "empty_guard", + "zero_cmp_guard", + "sp_null_guard", + ] { let a = &all[name]; assert_eq!(a.guarded_writes.len(), 1, "{name}: {:?}", a.guarded_writes); assert!( @@ -1833,6 +1928,11 @@ struct Cfg { ("present_guard", "ptr_"), ("bare_truth_guard", "ptr_"), ("not_empty_guard", "name_"), + // The member reached through parentheses and through an explicit + // `this`. Both must survive the subject check. + ("paren_truth_guard", "ptr_"), + ("this_truth_guard", "ptr_"), + ("sp_truth_guard", "sp_"), ] { let a = &all[name]; assert_eq!(a.guarded_writes, [member], "{name} guarded writes"); @@ -1894,6 +1994,70 @@ struct Cfg { ); } + /// Conditions that only MENTION the member. Each writes it in exactly the + /// branch the pre-fix rule guarded, because the misclassification is + /// silent whenever the write happens to land in the other branch — a + /// fixture with the write on the wrong side would pass without the fix. + const MENTIONS: &str = r#" +bool is_ready(int* p); +struct Sp { bool operator==(decltype(nullptr)) const; int operator+(int) const; explicit operator bool() const; }; +struct Cfg { + int* ptr_; + int count_; + Sp sp_; + // The predicate's value says nothing about whether `ptr_` is absent. + void helper_call(int* v) { if (is_ready(ptr_)) { } else { ptr_ = v; } } + void neg_helper_call(int* v) { if (!is_ready(ptr_)) { ptr_ = v; } } + // ... nor does a comparison of that predicate against a null literal. + void helper_cmp_null(int* v) { if (is_ready(ptr_) == 0) { ptr_ = v; } } + // Arithmetic on the member is not a test of the member. + void arithmetic(int v) { if (count_ + 1) { } else { count_ = v; } } + void neg_arithmetic(int v) { if (!(count_ + 1)) { count_ = v; } } + // A dereference tests the POINTEE; `ptr_` holds the same address either + // way, so neither branch proves it absent. + void deref_truth(int* v) { if (*ptr_) { } else { ptr_ = v; } } + void neg_deref(int* v) { if (!*ptr_) { ptr_ = v; } } + // An OVERLOADED arithmetic operator reaches the walker as a call whose + // first child is the left operand, so it takes a different path through + // the subject check than the builtin `+` above and needs its own case. + void overloaded_plus(Sp v) { if (sp_ + 1) { } else { sp_ = v; } } +}; +"#; + + /// The silence half of the subject check: a condition that merely mentions + /// the member is not a guard, so its write stays a plain write. + /// + /// Every one of these was recorded as a guarded write before + /// `member_is_condition_subject` existed, because the branch table reads + /// tokens across the whole condition and never asked where the member sat + /// in it. `guarded_writes` feeds `recipe::classify`, which prefers + /// `Default` over `Compute`/`Normalize`, so each was a method classified + /// by a guard the source does not contain. + #[test] + fn a_condition_that_only_mentions_the_member_is_not_a_guard() { + let all = arms("mentions", MENTIONS); + for name in [ + "helper_call", + "neg_helper_call", + "helper_cmp_null", + "arithmetic", + "neg_arithmetic", + "deref_truth", + "neg_deref", + "overloaded_plus", + ] { + let a = &all[name]; + // Anti-vacuity: the write must still be recorded, or this test + // would pass on an arm that saw nothing at all. + assert!(!a.writes.is_empty(), "{name} should still write"); + assert!( + a.guarded_writes.is_empty(), + "{name} claimed a guarded write: {:?}", + a.guarded_writes + ); + } + } + #[test] fn the_mutator_vocabulary_is_configurable() { let src = r#" From 038bbf8fbf745b51b03c5eeaf578c541c182b8c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 00:19:45 +0000 Subject: [PATCH 19/19] End the arguments array at the first bracket OUTSIDE a string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The array's closing bracket was found with `contains(']')`, so a `]` inside a JSON string ended collection early. `"-I/tmp/sdk]"` is a valid argument, and after it every remaining flag was dropped: libclang then parses the translation unit with a truncated flag list, which surfaces as a partial AST rather than an error, and the harvested events are wrong for a compilation database that is itself entirely valid. This is the same silent-truncation class as the comma split fixed in 5a9ea47, one level up. That one loses a single argument; this one loses every argument after it. Both new tests put the flag that gets lost — `-DTAIL=1` — AFTER the bracket-bearing one, since a fixture with nothing following it passes without the fix. `unquoted_bracket` shares its string/escape state machine with `json_string_array`, deliberately: a scanner that disagreed with the one doing the extraction would be a second source of truth about the same text. Both call sites use it — the multiline accumulator and the single-line form, which had the same bug and needed its own regression, because an array that opens and closes on one line takes a different branch. Written test-first: both cases failed with the truncation before the fix, and a disable that drops only the string tracking (returning the first `]` unconditionally) fails exactly those two and no others. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011JRoXmYPrE8KubXVrDB71L --- .../ruff_cpp_spo/examples/harvest_events.rs | 86 ++++++++++++++++++- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/ruff_cpp_spo/examples/harvest_events.rs b/crates/ruff_cpp_spo/examples/harvest_events.rs index b1ea5eabb40fb3..05b55c4ce90c06 100644 --- a/crates/ruff_cpp_spo/examples/harvest_events.rs +++ b/crates/ruff_cpp_spo/examples/harvest_events.rs @@ -137,10 +137,16 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { // raw text is accumulated and scanned for quoted runs at the closing // bracket, NEVER split on commas: `-DPAIR=std::pair` is one // valid argument, and splitting it would silently drop the flag. + // + // The array ends at the first `]` OUTSIDE a string. A bare + // `contains(']')` stops early on `-I/tmp/sdk]` and keeps only the + // flags before it — the same silent truncation as the comma split, one + // level up: that one loses an argument, this one loses every argument + // after it. if let Some(acc) = collecting.as_mut() { acc.push_str(line); - if line.contains(']') { - argv = Some(json_string_array(acc)); + if let Some(end) = unquoted_bracket(acc) { + argv = Some(json_string_array(&acc[..end])); collecting = None; } continue; @@ -152,8 +158,8 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { cmd = Some(v); } else if let Some(rest) = line.strip_prefix("\"arguments\":") { let rest = rest.trim_start(); - if rest.contains(']') { - argv = Some(json_string_array(rest)); + if let Some(end) = unquoted_bracket(rest) { + argv = Some(json_string_array(&rest[..end])); } else { collecting = Some(rest.to_string()); } @@ -183,6 +189,33 @@ fn parse_cc_json(text: &str) -> Vec<(PathBuf, Vec)> { /// comma inside a string is CONTENT: `["-DPAIR=std::pair"]` is one /// argument, and a comma split turns it into two invalid fragments that are /// then dropped, silently changing the flags a translation unit is parsed with. +/// The byte index of the first `]` that is NOT inside a JSON string, or `None` +/// when the array is still open. +/// +/// Shares its string/escape state machine with [`json_string_array`], so the +/// two agree on where a string starts and ends; a scanner that disagreed with +/// the one doing the extraction would be a second source of truth about the +/// same text. +fn unquoted_bracket(text: &str) -> Option { + let (mut inside, mut esc) = (false, false); + for (i, c) in text.char_indices() { + if inside { + if esc { + esc = false; + } else if c == '\\' { + esc = true; + } else if c == '"' { + inside = false; + } + } else if c == '"' { + inside = true; + } else if c == ']' { + return Some(i); + } + } + None +} + fn json_string_array(text: &str) -> Vec { let mut out = Vec::new(); let mut cur = String::new(); @@ -455,6 +488,51 @@ mod tests { assert_eq!(got.len(), 3, "a comma inside a string must not split it"); } + /// A `]` INSIDE a JSON string is content, not the end of the array. Ending + /// collection at the first bracket keeps only a prefix of the flags, and a + /// truncated flag list is a partial AST rather than an error — the same + /// silent-truncation class as the comma split above, one level up: that + /// one splits an argument, this one drops every argument after it. + #[test] + fn a_bracket_inside_an_argument_does_not_end_the_array() { + let db = r#"[ +{ + "directory": "/tmp", + "arguments": [ + "c++", + "-std=c++17", + "-I/tmp/sdk]", + "-DTAIL=1", + "-c", + "a.cpp" + ], + "file": "/tmp/a.cpp" +} +]"#; + let got = parse_cc_json(db); + assert_eq!(got.len(), 1, "one entry"); + // `-DTAIL=1` sits AFTER the bracket-bearing argument, so it is exactly + // what an early stop loses. + assert_eq!(got[0].1, ["-std=c++17", "-I/tmp/sdk]", "-DTAIL=1"]); + } + + /// The same truncation on ONE line: the array is still open after a + /// bracket that lives inside a string, so the remaining lines belong to it. + #[test] + fn a_bracket_inside_an_argument_does_not_close_a_single_line_array() { + let db = r#"[ +{ + "directory": "/tmp", + "arguments": ["c++", "-I/tmp/sdk]", + "-DTAIL=1", "-c", "a.cpp"], + "file": "/tmp/a.cpp" +} +]"#; + let got = parse_cc_json(db); + assert_eq!(got.len(), 1, "one entry"); + assert_eq!(got[0].1, ["-I/tmp/sdk]", "-DTAIL=1"]); + } + /// clang accepts `-I dir` as two tokens. Keeping the bare `-I` and dropping /// the directory hands libclang an incomplete flag and loses the include /// path, which shows up as a partial AST rather than an error.