From 8b581c345ad7642a54a0517b18944832e0823cc3 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Thu, 12 Nov 2026 20:25:41 +0100 Subject: [PATCH 01/18] refactor(bench): split bench_backends into bench_models and score_models bench_backends timed every compiled backend side by side, with ONNX as the reference. ONNX is no speed target, but it is a sound accuracy reference. So bench_models times the embedder and the reranker on one backend chosen with --backend (blas, or metal in a macOS build), and score_models scores blas and metal against ONNX, which it requires. Both now cover the Metal embedder, and neither runs bge-reranker-base, the speed baseline of the switch to gte. The workloads move to benchmark/workloads.rs, which both include. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +- Cargo.toml | 18 ++- README.md | 2 +- benchmark/README.md | 62 +++++--- benchmark/bench_backends.rs | 278 ------------------------------------ benchmark/bench_models.rs | 134 +++++++++++++++++ benchmark/score_models.rs | 100 +++++++++++++ benchmark/workloads.rs | 99 +++++++++++++ 8 files changed, 390 insertions(+), 308 deletions(-) delete mode 100644 benchmark/bench_backends.rs create mode 100644 benchmark/bench_models.rs create mode 100644 benchmark/score_models.rs create mode 100644 benchmark/workloads.rs diff --git a/AGENTS.md b/AGENTS.md index e2a4253d..f6c16526 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,5 +69,6 @@ encoder architecture: `bert.rs` (the embedders, and bge-reranker-base on the CPU (gte-reranker-modernbert-base, the reranker). `onnx` runs the same reranker, from the ONNX export in gte's Hub repo. CI lints and unit-tests the default build on Linux and on macOS 15, each with the other's seam compiled out, and `onnx` on Linux: so also run -`cargo clippy --all-targets --no-default-features --features onnx` before calling work done; -`cargo run --release --features onnx --example bench_backends` A/Bs them (latency + agreement). +`cargo clippy --all-targets --no-default-features --features onnx` before calling work done. +`cargo run --release --features onnx --example score_models` scores `blas` and `metal` against +`onnx`, and `cargo run --release --example bench_models -- --backend ` times one of them. diff --git a/Cargo.toml b/Cargo.toml index 5e8ff84f..829b94c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ metal = [ "dep:half", ] # ONNX inference backend (fastembed/ort). When both backends are compiled in, `blas` is the one -# funes runs; the ONNX types are the reference backend for bench_backends. +# funes runs, and the ONNX types are the reference score_models measures it against. onnx = ["dep:fastembed"] # Benchmarks live under benchmark/ alongside their README; run via `cargo run --example `. @@ -84,11 +84,19 @@ onnx = ["dep:fastembed"] name = "bench_recall" path = "benchmark/bench_recall.rs" -# A/B the inference backends (BLAS vs ONNX as reference). Needs both compiled in: -# cargo run --release --features onnx --example bench_backends +# Time the embedder and the reranker on one backend, blas or, in a macOS build, metal: +# cargo run --release --example bench_models -- --backend metal [[example]] -name = "bench_backends" -path = "benchmark/bench_backends.rs" +name = "bench_models" +path = "benchmark/bench_models.rs" +required-features = ["blas"] + +# Score the backends against ONNX Runtime, which it needs compiled in: +# cargo run --release --features onnx --example score_models +[[example]] +name = "score_models" +path = "benchmark/score_models.rs" +required-features = ["onnx"] # A default `funes recall` scored against a labeled set of questions. [[example]] diff --git a/README.md b/README.md index 27e640d1..082db381 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ back to the CPU where no GPU starts. An ONNX Runtime backend is available as an ```bash cargo build --release --no-default-features --features onnx # ONNX backend instead -cargo run --release --features onnx --example bench_backends # A/B both backends +cargo run --release --features onnx --example score_models # score the built-in backend against it ``` ## Notes diff --git a/benchmark/README.md b/benchmark/README.md index 22f1d0a5..f12fd367 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,9 +1,11 @@ # Benchmarks -Four runnable examples, each `cargo run --release --example `: +Five runnable examples, each `cargo run --release --example `: - **`bench_recall`** — `recall()` latency, local vs remote, cold vs warm. -- **`bench_backends`** — latency and output agreement between the inference backends. +- **`bench_models`** — embedding and reranking time on one inference backend. +- **`score_models`** — how closely the inference backends reproduce ONNX Runtime's embeddings and + reranker scores. - **`score_recall`** — `funes recall` scored against a labeled set of questions. - **`sweep_rerank`** — how much reranking helps at each pool size, on a `dacorvo/funes-recall-benchmark` set. @@ -89,34 +91,50 @@ that floor moves with the CPU and the inference backend. What the benchmark meas - For a private dataset, a token must be available (`HF_TOKEN` or the cached login) — the same one recall uses. -## `bench_backends` — inference backend comparison +## `bench_models` — embedding and reranking time -`bench_backends.rs` compares every compiled `Embedder`/`Reranker` implementation. Build with both -the default BLAS backend and the optional ONNX reference: +`bench_models.rs` times funes's embedder and reranker (gte-reranker-modernbert-base) on one +backend: `--backend blas`, the default, runs the CPU forwards every build carries, and +`--backend metal` the GPU forwards of a macOS build. `--model` names the embedding model, +`BAAI/bge-small-en-v1.5` when omitted. ```sh -cargo run --release --features onnx --example bench_backends -cargo run --release --features onnx --example bench_backends -- intfloat/multilingual-e5-small +cargo run --release --example bench_models -- --backend metal +cargo run --release --example bench_models -- --model intfloat/multilingual-e5-small ``` -It runs seven fixed workloads: 16 short documents to expose per-call overhead; 30 documents near -the 512-token truncation limit to approximate recall's rerank worst case; a ragged batch of 2 capped -documents among 14 short ones; 32 and 100 documents whose token lengths are the quantiles of real -chunks (100 is recall's pool at `--candidates 100`); and 256-document mixed and real batches, the -production batch size, which show how a backend scales and what it spends on padding. +It times the forwards themselves. funes runs them behind a router that keeps a call on the CPU +until it holds 32 texts or 8 pairs, and the two backends' startup and first calls are what those +thresholds weigh. Each model's startup comes first: how long it took to load, then its first call +on the 32 real documents, which a one-shot process such as `funes recall` pays in full. Then each +workload's time per call and documents per second, after one warm-up call. -Rerankers are compared within a model: gte-reranker-modernbert-base on ONNX, BLAS and, on macOS, -Metal, and bge-reranker-base on BLAS as the latency baseline. Each reranker's startup comes first: how -long it took to load, then its first call on the 32 real documents, which a one-shot `funes recall` -pays in full. Then each workload's latency, and agreement with the first backend of the same model: +The seven workloads live in `workloads.rs`, which `score_models` shares: -- `cos↔ref` is the minimum cosine similarity between corresponding embedding vectors. -- `Δ↔ref` is the maximum absolute difference between corresponding reranker scores. +- 16 short documents, to expose per-call overhead. +- 30 documents near the 512-token truncation limit, recall's rerank worst case. +- A ragged batch of 2 capped documents among 14 short ones. +- 32 and 100 documents whose token lengths are the quantiles of real chunks. 100 is recall's pool + at `--candidates 100`. +- 256-document mixed and real batches, the production batch size, which show how a backend scales + and what it spends on padding. -ONNX is the reference when that feature is present, BLAS otherwise. The one argument names the -embedding model, `BAAI/bge-small-en-v1.5` when omitted. Edit the fixed query or workloads when investigating a -particular regression. Building with only one backend still prints its timing, along with a note -that there is nothing to compare. +Edit the query or the workloads when investigating a particular regression. + +## `score_models` — the backends against ONNX + +`score_models.rs` measures how closely funes's backends reproduce ONNX Runtime running each model's +ONNX export: the CPU forwards (`blas`) and, in a macOS build, the GPU forwards (`metal`), over the +same seven workloads. ONNX is the reference, so the example needs it compiled in: + +```sh +cargo run --release --features onnx --example score_models +cargo run --release --features onnx --example score_models -- --model intfloat/multilingual-e5-small +``` + +- `cos↔onnx` is the lowest cosine similarity between a document's embedding on the backend and on + ONNX. +- `Δ↔onnx` is the largest absolute difference between their reranker scores. ## `score_recall` — recall against labeled questions diff --git a/benchmark/bench_backends.rs b/benchmark/bench_backends.rs deleted file mode 100644 index 69445f84..00000000 --- a/benchmark/bench_backends.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! A/B the inference backends behind the `Embedder`/`Reranker` traits: latency + agreement (does a -//! faster backend embed/rank the same as the reference?). Backend-agnostic and cross-platform — it -//! compares whatever backends are compiled in, with ONNX (fastembed) as the reference when present: -//! cargo run --release --features onnx --example bench_backends [-- ] -//! -//! Rerankers are compared within a model: gte-reranker-modernbert-base on ONNX (the reference), -//! BLAS and, on macOS, Metal, and bge-reranker-base on BLAS as the latency baseline. Each -//! reranker's startup comes first: loading it, then its first call, which a one-shot `funes recall` -//! pays in full (the GPU compiles a graph per batch shape on first use). -//! -//! Seven workloads, because they stress different things: a batch of short docs is dominated by -//! per-call overheads (tokenization, thread spawns), while 30 docs at the 512-token truncation -//! cap — recall's rerank worst case — is dominated by GEMM throughput and memory behavior. The -//! ragged batch is real indexing's shape — batch-longest padding masks most attention columns, -//! whose softmax weights underflow, and the scores×V GEMM then reads what they leave behind. The -//! real batch spreads its lengths the way measured chunks do, so it shows what a backend spends -//! on padding a mixed batch. The 256-document workloads match the production batch size and show -//! how grouping by length scales. The 100-document one is recall's pool at `--candidates 100`. -//! -//! Adding a backend = impl Embedder or Reranker, gate it behind a feature, and push it in -//! `embedders()` or `rerankers()`. - -use std::time::Instant; - -use anyhow::{anyhow, Result}; -use funes::inference::{Embedder, EmbeddingModel, Reranker}; - -const QUERY: &str = "why did we move the reranker off the onnx runtime"; - -fn short_docs() -> Vec { - [ - "the reranker is a cross-encoder scoring query-passage pairs jointly.", - "onnx runtime uses MLAS on the CPU, which on Apple Silicon runs on NEON, not AMX.", - "a hand-written forward calls Accelerate cblas_sgemm, which reaches the AMX matrix units.", - "the embedding model is bge-small-en-v1.5, a 384-dimensional BERT sentence encoder.", - "recall fuses vector ANN and BM25 hits by reciprocal rank before reranking.", - "the memory is a lance dataset with an IVF_PQ vector index and a full-text index.", - "candle's metal backend was missing a layer-norm kernel, so it could not run the model.", - "the cat knocked a glass off the counter and it shattered on the floor.", - "quarterly revenue rose twelve percent on strong subscription renewals.", - "tokenization uses the huggingface tokenizers crate loading the model's tokenizer.json.", - "softmax over the attention scores uses a vectorized exp from the platform seam.", - "the recipe needs two cups of flour, a teaspoon of salt, and three eggs.", - "fp8 has no hardware path on apple silicon, so int8 is the only accelerated low-precision.", - "attention masks let the transformer ignore padding tokens in a ragged batch.", - "the marathon route winds through six neighborhoods before the riverside finish.", - "hf-hub fetches whole files because the xet cdn taxes every byte-range read.", - ] - .map(String::from) - .to_vec() -} - -const SENT: &str = "recall fuses vector ann and bm25 hits by reciprocal rank before the cross-encoder \ - rescores each candidate against the query using joint attention over the pair. "; - -fn capped_docs(n: usize) -> Vec { - // ~500 tokens after tokenization, truncated at 512 — recall's rerank candidates at the cap. - let doc = SENT.repeat(18); - vec![doc; n] -} - -fn long_docs() -> Vec { - capped_docs(30) -} - -/// A ragged batch — a couple of ~512-token docs, the rest short: real indexing's shape, and the -/// one that exposes padding stalls. -fn mixed_docs() -> Vec { - let short = short_docs(); - let mut docs = capped_docs(2); - docs.extend((0..16 - docs.len()).map(|i| short[i % short.len()].clone())); - docs -} - -/// Words of `SENT` whose embedder token counts land on the 32 quantiles of real chunk lengths, -/// measured over 12,971 chunks of 100 Claude Code sessions: 21 up to the 512 cap, median 314. -const REAL_WORDS: [usize; 32] = [ - 13, 30, 38, 51, 66, 87, 106, 126, 145, 163, 176, 187, 196, 205, 213, 218, 226, 232, 238, 247, 255, 262, 269, 280, - 288, 296, 306, 314, 327, 340, 362, 363, -]; - -/// One doc per quantile of real chunk lengths, in a scrambled order — the length spread indexing -/// hands `embed`, and about recall's 30 rerank candidates. -fn real_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..32).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() -} - -/// Like `mixed_docs` but 256 documents — the batch size `embed_batched` uses in production. -fn big_mixed_docs() -> Vec { - let short = short_docs(); - let mut docs = capped_docs(32); - docs.extend((0..256 - docs.len()).map(|i| short[i % short.len()].clone())); - docs -} - -/// Like `real_docs` but 256 documents — the production batch size, with the same length spread. -fn big_real_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..256).map(|i| words[..REAL_WORDS[i % 32]].join(" ")).collect() -} - -/// Like `real_docs` but 100 documents: recall's pool at `--candidates 100`. -fn deep_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..100).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() -} - -/// The embedders compiled in, ONNX first when it is: the first is the agreement reference. -fn embedders(model: EmbeddingModel) -> Result)>> { - let mut v: Vec<(&'static str, Box)> = Vec::new(); - #[cfg(feature = "onnx")] - { - v.push(("onnx", Box::new(funes::inference::OnnxEmbedder::new(model)?))); - } - #[cfg(feature = "blas")] - { - v.push(("blas", Box::new(funes::inference::blas::BlasEmbedder::new(model)?))); - } - Ok(v) -} - -/// A reranker under test, and how long it took to load. -struct Ranker { - model: &'static str, - backend: &'static str, - rr: Box, - load_ms: f64, -} - -/// The rerankers compiled in, by model: the first of a model is the reference for the others. -fn rerankers() -> Result> { - fn load( - model: &'static str, - backend: &'static str, - make: impl FnOnce() -> Result>, - ) -> Result { - let t = Instant::now(); - let rr = make()?; - let load_ms = t.elapsed().as_secs_f64() * 1000.0; - Ok(Ranker { - model, - backend, - rr, - load_ms, - }) - } - let mut v = Vec::new(); - #[cfg(feature = "onnx")] - v.push(load("gte", "onnx", || { - Ok(Box::new(funes::inference::OnnxReranker::new()?)) - })?); - #[cfg(feature = "blas")] - v.push(load("gte", "blas", || { - Ok(Box::new(funes::inference::blas::GteReranker::new()?)) - })?); - #[cfg(all(feature = "metal", target_os = "macos"))] - v.push(load("gte", "metal", || { - Ok(Box::new(funes::inference::metal::MetalReranker::new()?)) - })?); - #[cfg(feature = "blas")] - v.push(load("bge", "blas", || { - Ok(Box::new(funes::inference::blas::BlasReranker::new()?)) - })?); - Ok(v) -} - -fn time(warmups: usize, iters: usize, mut f: F) -> f64 { - for _ in 0..warmups { - f(); - } - let t = Instant::now(); - for _ in 0..iters { - f(); - } - t.elapsed().as_secs_f64() * 1000.0 / iters as f64 -} - -fn cosine(a: &[f32], b: &[f32]) -> f32 { - let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); - let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); - let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); - dot / (na * nb) -} - -fn main() -> Result<()> { - let model = match std::env::args().nth(1) { - Some(id) => EmbeddingModel::from_id(&id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, - None => EmbeddingModel::BgeSmallEn, - }; - let mut embs = embedders(model)?; - let mut rrs = rerankers()?; - if embs.len() < 2 { - eprintln!("note: only one backend is compiled — build with `--features onnx` to A/B against the reference\n"); - } - - // (label, docs, warmups, timed iters) — few iterations where a single forward runs seconds, - // except the ragged batch: it carries the padding behavior worth measuring, and one timed - // iteration cannot resolve a difference of a few percent. - let workloads = [ - ("16×short", short_docs(), 2, 5), - ("30×~500tok", long_docs(), 1, 3), - ("16×mixed", mixed_docs(), 1, 5), - ("32×real", real_docs(), 1, 3), - ("100×real", deep_docs(), 1, 3), - ("256×mixed", big_mixed_docs(), 1, 1), - ("256×real", big_real_docs(), 1, 1), - ]; - - let real = real_docs(); - let real: Vec<&str> = real.iter().map(String::as_str).collect(); - println!("reranker startup: load, then a first call on 32×real"); - println!( - "{:<6} {:<8} {:>10} {:>14}", - "model", "backend", "load ms", "first call ms" - ); - for r in rrs.iter_mut() { - let t = Instant::now(); - r.rr.rerank(QUERY, &real)?; - let first_ms = t.elapsed().as_secs_f64() * 1000.0; - println!("{:<6} {:<8} {:>10.0} {:>14.0}", r.model, r.backend, r.load_ms, first_ms); - } - - println!("\nembedding: {}", model.id()); - println!( - "{:<12} {:<8} {:>10} {:>16}", - "workload", "backend", "embed ms", "cos↔ref" - ); - for (label, docs, warmups, iters) in &workloads { - let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); - // Run each backend once for correctness, then time it. Reference = the first backend. - let mut reference: Option>> = None; - for (name, emb) in embs.iter_mut() { - let e = emb.embed(&docs)?; - let ms = time(*warmups, *iters, || { - emb.embed(&docs).unwrap(); - }); - let cos = match &reference { - None => "(ref)".to_string(), - Some(r) => { - let min = e.iter().zip(r).map(|(a, b)| cosine(a, b)).fold(1f32, f32::min); - format!("{min:.6}") - } - }; - println!("{label:<12} {name:<8} {ms:>10.1} {cos:>16}"); - reference.get_or_insert(e); - } - } - - println!("\nreranking"); - println!( - "{:<12} {:<6} {:<8} {:>10} {:>16}", - "workload", "model", "backend", "rerank ms", "Δ↔ref" - ); - for (label, docs, warmups, iters) in &workloads { - let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); - // Run each reranker once for correctness, then time it. Reference = the model's first. - let mut references: Vec<(&str, Vec)> = Vec::new(); - for r in rrs.iter_mut() { - let scores = r.rr.rerank(QUERY, &docs)?; - let ms = time(*warmups, *iters, || { - r.rr.rerank(QUERY, &docs).unwrap(); - }); - let diff = match references.iter().find(|(m, _)| *m == r.model) { - None => "(ref)".to_string(), - Some((_, want)) => { - let d = scores.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); - format!("{d:.6}") - } - }; - println!("{label:<12} {:<6} {:<8} {ms:>10.1} {diff:>16}", r.model, r.backend); - if !references.iter().any(|(m, _)| *m == r.model) { - references.push((r.model, scores)); - } - } - } - Ok(()) -} diff --git a/benchmark/bench_models.rs b/benchmark/bench_models.rs new file mode 100644 index 00000000..8c1ebe33 --- /dev/null +++ b/benchmark/bench_models.rs @@ -0,0 +1,134 @@ +//! Time funes's embedder and reranker on one inference backend: +//! +//! ```text +//! cargo run --release --example bench_models -- --backend metal +//! ``` +//! +//! `--backend blas` (the default) times the CPU forwards every build carries, `--backend metal` the +//! GPU forwards of a macOS build. These are the forwards themselves: funes runs them behind a router +//! that keeps a call on the CPU until it holds `GPU_MIN_TEXTS` texts or `GPU_MIN_PAIRS` pairs, and +//! the two backends' startup and first calls are what those thresholds weigh. Each model's startup +//! comes first: loading it, then its first call, which a one-shot process pays in full (the GPU +//! uploads its weights and compiles a graph per batch shape on first use). Then each workload's time +//! per call, after one warm-up call. + +#[path = "workloads.rs"] +mod workloads; + +use std::time::Instant; + +use anyhow::{anyhow, Result}; +use clap::{Parser, ValueEnum}; +use funes::inference::{Embedder, EmbeddingModel, Reranker}; + +#[derive(Parser)] +struct Args { + /// The backend whose forwards are timed. + #[arg(long, value_enum, default_value_t = Backend::Blas)] + backend: Backend, + /// The embedding model, by its Hub id. `BAAI/bge-small-en-v1.5` when omitted. + #[arg(long)] + model: Option, +} + +#[derive(Clone, Copy, ValueEnum)] +enum Backend { + /// The CPU forwards, in every build. + Blas, + /// The GPU forwards, in a macOS build. + #[cfg(all(feature = "metal", target_os = "macos"))] + Metal, +} + +impl Backend { + fn name(self) -> &'static str { + match self { + Backend::Blas => "blas", + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => "metal", + } + } + + fn embedder(self, model: EmbeddingModel) -> Result> { + Ok(match self { + Backend::Blas => Box::new(funes::inference::blas::BlasEmbedder::new(model)?), + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => Box::new(funes::inference::metal::MetalEmbedder::new(model)?), + }) + } + + fn reranker(self) -> Result> { + Ok(match self { + Backend::Blas => Box::new(funes::inference::blas::GteReranker::new()?), + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => Box::new(funes::inference::metal::MetalReranker::new()?), + }) + } +} + +/// Timed calls for a workload of `n` documents: fewer where a call runs seconds. Up to 16 documents +/// get 5, since the ragged batch carries the padding behavior worth measuring and one call cannot +/// resolve a difference of a few percent. +fn iterations(n: usize) -> usize { + match n { + 0..=16 => 5, + 17..=100 => 3, + _ => 1, + } +} + +/// What `f` returns, and the milliseconds it took. +fn timed(f: impl FnOnce() -> Result) -> Result<(T, f64)> { + let t = Instant::now(); + let out = f()?; + Ok((out, t.elapsed().as_secs_f64() * 1000.0)) +} + +/// Milliseconds per call of `f` over `iters` calls, after one warm-up call. +fn per_call(iters: usize, mut f: impl FnMut() -> Result<()>) -> Result { + f()?; + let ((), ms) = timed(|| (0..iters).try_for_each(|_| f()))?; + Ok(ms / iters as f64) +} + +fn main() -> Result<()> { + let args = Args::parse(); + let model = match &args.model { + Some(id) => EmbeddingModel::from_id(id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, + None => EmbeddingModel::BgeSmallEn, + }; + let real = workloads::real_docs(); + let real: Vec<&str> = real.iter().map(String::as_str).collect(); + + let (mut emb, emb_load) = timed(|| args.backend.embedder(model))?; + let (_, emb_first) = timed(|| emb.embed(&real))?; + let (mut rr, rr_load) = timed(|| args.backend.reranker())?; + let (_, rr_first) = timed(|| rr.rerank(workloads::QUERY, &real))?; + println!( + "{} backend: {} and gte-reranker-modernbert-base", + args.backend.name(), + model.id() + ); + println!("\nstartup: load, then a first call on 32×real"); + println!("{:<10} {:>10} {:>14}", "model", "load ms", "first call ms"); + println!("{:<10} {emb_load:>10.0} {emb_first:>14.0}", "embedder"); + println!("{:<10} {rr_load:>10.0} {rr_first:>14.0}", "reranker"); + + println!( + "\n{:<12} {:>10} {:>10} {:>11} {:>10}", + "workload", "embed ms", "texts/s", "rerank ms", "pairs/s" + ); + for (label, docs) in workloads::workloads() { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let iters = iterations(docs.len()); + let embed_ms = per_call(iters, || emb.embed(&docs).map(drop))?; + let rerank_ms = per_call(iters, || rr.rerank(workloads::QUERY, &docs).map(drop))?; + let per_s = |ms: f64| docs.len() as f64 * 1000.0 / ms; + println!( + "{label:<12} {embed_ms:>10.1} {:>10.0} {rerank_ms:>11.1} {:>10.0}", + per_s(embed_ms), + per_s(rerank_ms) + ); + } + Ok(()) +} diff --git a/benchmark/score_models.rs b/benchmark/score_models.rs new file mode 100644 index 00000000..f7ea3a08 --- /dev/null +++ b/benchmark/score_models.rs @@ -0,0 +1,100 @@ +//! Score funes's inference backends against ONNX Runtime running each model's ONNX export: how +//! closely the CPU forwards (`blas`) and, in a macOS build, the GPU forwards (`metal`) reproduce its +//! embeddings and its reranker scores, over the workloads `bench_models` times. ONNX is the +//! reference, so the example needs it compiled in: +//! +//! ```text +//! cargo run --release --features onnx --example score_models +//! ``` +//! +//! `cos↔onnx` is the lowest cosine similarity between a document's embedding on the backend and on +//! ONNX, and `Δ↔onnx` the largest absolute difference between their reranker scores. + +#[path = "workloads.rs"] +mod workloads; + +use anyhow::{anyhow, Result}; +use clap::Parser; +use funes::inference::{Embedder, EmbeddingModel, OnnxEmbedder, OnnxReranker, Reranker}; + +#[derive(Parser)] +struct Args { + /// The embedding model, by its Hub id. `BAAI/bge-small-en-v1.5` when omitted. + #[arg(long)] + model: Option, +} + +/// A backend scored against ONNX: its name, its embedder and its reranker. +type Backend = (&'static str, Box, Box); + +/// The backends compiled in besides ONNX. +#[cfg(feature = "blas")] +fn backends(model: EmbeddingModel) -> Result> { + Ok(vec![ + ( + "blas", + Box::new(funes::inference::blas::BlasEmbedder::new(model)?) as Box, + Box::new(funes::inference::blas::GteReranker::new()?) as Box, + ), + #[cfg(all(feature = "metal", target_os = "macos"))] + ( + "metal", + Box::new(funes::inference::metal::MetalEmbedder::new(model)?) as Box, + Box::new(funes::inference::metal::MetalReranker::new()?) as Box, + ), + ]) +} + +/// A build with ONNX alone has no other backend to score. +#[cfg(not(feature = "blas"))] +fn backends(_: EmbeddingModel) -> Result> { + Ok(Vec::new()) +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); + let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); + let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); + dot / (na * nb) +} + +fn main() -> Result<()> { + let args = Args::parse(); + let model = match &args.model { + Some(id) => EmbeddingModel::from_id(id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, + None => EmbeddingModel::BgeSmallEn, + }; + let mut backends = backends(model)?; + if backends.is_empty() { + println!("nothing to score: this build has no backend besides ONNX"); + return Ok(()); + } + let mut onnx_embedder = OnnxEmbedder::new(model)?; + let mut onnx_reranker = OnnxReranker::new()?; + let workloads = workloads::workloads(); + + println!("embedding: {}", model.id()); + println!("{:<12} {:<8} {:>12}", "workload", "backend", "cos↔onnx"); + for (label, docs) in &workloads { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let want = onnx_embedder.embed(&docs)?; + for (name, embedder, _) in backends.iter_mut() { + let got = embedder.embed(&docs)?; + let min = got.iter().zip(&want).map(|(a, b)| cosine(a, b)).fold(1f32, f32::min); + println!("{label:<12} {name:<8} {min:>12.6}"); + } + } + + println!("\nreranking: gte-reranker-modernbert-base"); + println!("{:<12} {:<8} {:>12}", "workload", "backend", "Δ↔onnx"); + for (label, docs) in &workloads { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let want = onnx_reranker.rerank(workloads::QUERY, &docs)?; + for (name, _, reranker) in backends.iter_mut() { + let got = reranker.rerank(workloads::QUERY, &docs)?; + let max = got.iter().zip(&want).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + println!("{label:<12} {name:<8} {max:>12.6}"); + } + } + Ok(()) +} diff --git a/benchmark/workloads.rs b/benchmark/workloads.rs new file mode 100644 index 00000000..02c9a369 --- /dev/null +++ b/benchmark/workloads.rs @@ -0,0 +1,99 @@ +//! The documents `bench_models` times and `score_models` scores, in seven workloads, because they +//! stress different things. A batch of short docs is dominated by per-call overheads (tokenization, +//! thread spawns), while 30 docs at the 512-token truncation cap, recall's rerank worst case, are +//! dominated by GEMM throughput and memory behavior. The ragged batch is real indexing's shape: +//! batch-longest padding masks most attention columns, whose softmax weights underflow, and the +//! scores×V GEMM then reads what they leave behind. The real batch spreads its lengths the way +//! measured chunks do, so it shows what a backend spends on padding a mixed batch. The 256-document +//! workloads match the production batch size and show how grouping by length scales. The +//! 100-document one is recall's pool at `--candidates 100`. + +/// The query every reranker workload scores its documents against. +pub const QUERY: &str = "why did we move the reranker off the onnx runtime"; + +fn short_docs() -> Vec { + [ + "the reranker is a cross-encoder scoring query-passage pairs jointly.", + "onnx runtime uses MLAS on the CPU, which on Apple Silicon runs on NEON, not AMX.", + "a hand-written forward calls Accelerate cblas_sgemm, which reaches the AMX matrix units.", + "the embedding model is bge-small-en-v1.5, a 384-dimensional BERT sentence encoder.", + "recall fuses vector ANN and BM25 hits by reciprocal rank before reranking.", + "the memory is a lance dataset with an IVF_PQ vector index and a full-text index.", + "candle's metal backend was missing a layer-norm kernel, so it could not run the model.", + "the cat knocked a glass off the counter and it shattered on the floor.", + "quarterly revenue rose twelve percent on strong subscription renewals.", + "tokenization uses the huggingface tokenizers crate loading the model's tokenizer.json.", + "softmax over the attention scores uses a vectorized exp from the platform seam.", + "the recipe needs two cups of flour, a teaspoon of salt, and three eggs.", + "fp8 has no hardware path on apple silicon, so int8 is the only accelerated low-precision.", + "attention masks let the transformer ignore padding tokens in a ragged batch.", + "the marathon route winds through six neighborhoods before the riverside finish.", + "hf-hub fetches whole files because the xet cdn taxes every byte-range read.", + ] + .map(String::from) + .to_vec() +} + +const SENT: &str = "recall fuses vector ann and bm25 hits by reciprocal rank before the cross-encoder \ + rescores each candidate against the query using joint attention over the pair. "; + +/// `n` documents of about 500 tokens, truncated at 512: recall's rerank candidates at the cap. +fn capped_docs(n: usize) -> Vec { + vec![SENT.repeat(18); n] +} + +/// A ragged batch, two ~512-token docs and the rest short: real indexing's shape, and the one that +/// exposes padding stalls. +fn mixed_docs() -> Vec { + let short = short_docs(); + let mut docs = capped_docs(2); + docs.extend((0..16 - docs.len()).map(|i| short[i % short.len()].clone())); + docs +} + +/// Words of `SENT` whose embedder token counts land on the 32 quantiles of real chunk lengths, +/// measured over 12,971 chunks of 100 Claude Code sessions: 21 up to the 512 cap, median 314. +const REAL_WORDS: [usize; 32] = [ + 13, 30, 38, 51, 66, 87, 106, 126, 145, 163, 176, 187, 196, 205, 213, 218, 226, 232, 238, 247, 255, 262, 269, 280, + 288, 296, 306, 314, 327, 340, 362, 363, +]; + +/// One doc per quantile of real chunk lengths, in a scrambled order: the length spread indexing +/// hands `embed`, and about recall's 30 rerank candidates. +pub fn real_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..32).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() +} + +/// Like `mixed_docs` but 256 documents, the batch size `embed_batched` uses in production. +fn big_mixed_docs() -> Vec { + let short = short_docs(); + let mut docs = capped_docs(32); + docs.extend((0..256 - docs.len()).map(|i| short[i % short.len()].clone())); + docs +} + +/// Like `real_docs` but 256 documents, the production batch size, with the same length spread. +fn big_real_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..256).map(|i| words[..REAL_WORDS[i % 32]].join(" ")).collect() +} + +/// Like `real_docs` but 100 documents: recall's pool at `--candidates 100`. +fn deep_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..100).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() +} + +/// Each workload's label and documents. +pub fn workloads() -> Vec<(&'static str, Vec)> { + vec![ + ("16×short", short_docs()), + ("30×~500tok", capped_docs(30)), + ("16×mixed", mixed_docs()), + ("32×real", real_docs()), + ("100×real", deep_docs()), + ("256×mixed", big_mixed_docs()), + ("256×real", big_real_docs()), + ] +} From 35b861077d750cb9090564ee4ab999535df0b6a4 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Thu, 12 Nov 2026 23:01:52 +0100 Subject: [PATCH 02/18] refactor(inference): drop bge-reranker-base funes reranks with gte-reranker-modernbert-base on every backend, and bench_backends was the last thing that ran bge-reranker-base, as the speed baseline of the switch to gte. The BERT encoder's XLM-R position offsets and roberta. weight prefix go with it, since only that model used them. The embedders still match ONNX as before in score_models. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/inference/blas.rs | 6 +- src/inference/blas/bert.rs | 112 +++++-------------------------------- 3 files changed, 17 insertions(+), 103 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f6c16526..51aa97e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,7 +65,7 @@ default `blas` (src/inference/blas.rs, hand-written forwards on Accelerate/faer) (fastembed/ort). On macOS the default build also carries `metal` (src/inference/metal.rs): the same encoders as MPSGraphs in fp16, which the embedder hands calls of 32+ texts to and the reranker calls of 8+ pairs, falling back to `blas` where no GPU starts. `blas` and `metal` each keep one module per -encoder architecture: `bert.rs` (the embedders, and bge-reranker-base on the CPU) and `modernbert.rs` +encoder architecture: `bert.rs` (the embedders) and `modernbert.rs` (gte-reranker-modernbert-base, the reranker). `onnx` runs the same reranker, from the ONNX export in gte's Hub repo. CI lints and unit-tests the default build on Linux and on macOS 15, each with the other's seam compiled out, and `onnx` on Linux: so also run diff --git a/src/inference/blas.rs b/src/inference/blas.rs index 2156a3d5..44e991b5 100644 --- a/src/inference/blas.rs +++ b/src/inference/blas.rs @@ -3,8 +3,8 @@ //! wires it to Accelerate (cblas_sgemm on AMX + vForce vvexpf), Linux to faer's pure-Rust GEMM //! plus a runtime-dispatched polynomial exp. This module holds the seam, the kernels, the model //! loaders and the length-grouped batching. The encoders built on them are cross-platform source: -//! BERT (the embedders and bge-reranker-base) in [`bert`], ModernBERT -//! (gte-reranker-modernbert-base) in [`modernbert`]. Gated behind the `blas` feature. +//! BERT (the embedders) in [`bert`], ModernBERT (gte-reranker-modernbert-base) in [`modernbert`]. +//! Gated behind the `blas` feature. use std::collections::HashMap; use std::io::{Read, Seek, SeekFrom}; @@ -17,7 +17,7 @@ use tokenizers::{Encoding, PaddingParams, Tokenizer, TruncationParams}; pub(super) mod bert; pub(super) mod modernbert; -pub use bert::{BlasEmbedder, BlasReranker}; +pub use bert::BlasEmbedder; pub use modernbert::GteReranker; // --------------------------------------------------------------------------------------------- diff --git a/src/inference/blas/bert.rs b/src/inference/blas/bert.rs index 7be4a43e..b4405a0a 100644 --- a/src/inference/blas/bert.rs +++ b/src/inference/blas/bert.rs @@ -1,6 +1,6 @@ -//! The BERT encoder on the CPU, for the embedders and bge-reranker-base: post-norm blocks with -//! biases, and learned positions added to the embeddings. It shares the seam, the kernels, the -//! loaders and the length-grouped batching of [`super`]. +//! The BERT encoder on the CPU, for the embedders: post-norm blocks with biases, and learned +//! positions added to the embeddings. It shares the seam, the kernels, the loaders and the +//! length-grouped batching of [`super`]. use std::collections::HashMap; @@ -9,22 +9,18 @@ use tokenizers::Tokenizer; use super::{ by_length, ceil_div, gelu, hf_snapshot, l2_normalize, layernorm, linear, load_tokenizer, load_weights, mean_pool, - par_add, seam, seq_workers, softmax_rows, NO_T, RERANK_GROUP, T, + par_add, seam, seq_workers, softmax_rows, NO_T, T, }; -use crate::inference::{Embedder, EmbeddingModel, Reranker}; +use crate::inference::{Embedder, EmbeddingModel}; -const PAD: i64 = 1; - -/// A BERT-family encoder config. Reranker is XLM-R (pad-offset positions); embedder is BERT. +/// The BERT encoder's shape. #[derive(Clone, Copy)] struct Cfg { - prefix: &'static str, h: usize, heads: usize, hd: usize, ffn: usize, layers: usize, - bert_pos: bool, /// Sequences per forward. A forward pads every sequence to the longest it holds, so /// [`by_length`] runs inputs length-sorted, this many at a time. Measured on real chunk /// lengths: fewer shrinks the GEMMs, more brings the padding back. `Scratch` is sized to one @@ -32,24 +28,12 @@ struct Cfg { group: usize, } -const RERANK: Cfg = Cfg { - prefix: "roberta.", - h: 768, - heads: 12, - hd: 64, - ffn: 3072, - layers: 12, - bert_pos: false, - group: RERANK_GROUP, -}; const EMBED: Cfg = Cfg { - prefix: "", h: 384, heads: 12, hd: 32, ffn: 1536, layers: 12, - bert_pos: true, group: 16, }; @@ -73,9 +57,9 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u let g = |k: &str| w.get(k).unwrap_or_else(|| panic!("missing weight {k}")).as_slice(); let (h, ffn, hd, heads) = (c.h, c.ffn, c.hd, c.heads); let m = n * l; - let word = g(&format!("{}embeddings.word_embeddings.weight", c.prefix)); - let pos = g(&format!("{}embeddings.position_embeddings.weight", c.prefix)); - let typ = g(&format!("{}embeddings.token_type_embeddings.weight", c.prefix)); + let word = g("embeddings.word_embeddings.weight"); + let pos = g("embeddings.position_embeddings.weight"); + let typ = g("embeddings.token_type_embeddings.weight"); for (buf, len) in [ (&mut s.hid, m * h), (&mut s.q, m * h), @@ -94,20 +78,11 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u scope.spawn(move || { for (si, seq) in grp.chunks_mut(l * h).enumerate() { let sq = gi * seqs_per + si; - let mut run = 0i64; for i in 0..l { let id = ids[sq * l + i]; - let posid = if c.bert_pos { - i - } else if id != PAD { - run += 1; - (PAD + run) as usize - } else { - PAD as usize - }; let dst = &mut seq[i * h..i * h + h]; let we = &word[id as usize * h..id as usize * h + h]; - let pe = &pos[posid * h..posid * h + h]; + let pe = &pos[i * h..i * h + h]; for cc in 0..h { dst[cc] = we[cc] + pe[cc] + typ[cc]; } @@ -119,13 +94,13 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u layernorm( &mut s.hid, h, - g(&format!("{}embeddings.LayerNorm.weight", c.prefix)), - g(&format!("{}embeddings.LayerNorm.bias", c.prefix)), + g("embeddings.LayerNorm.weight"), + g("embeddings.LayerNorm.bias"), ); let scale = 1.0 / (hd as f32).sqrt(); for ly in 0..c.layers { - let p = format!("{}encoder.layer.{ly}", c.prefix); + let p = format!("encoder.layer.{ly}"); linear( &s.hid, m, @@ -289,64 +264,3 @@ impl Embedder for BlasEmbedder { }) } } - -/// bge-reranker-base cross-encoder: XLM-R encoder → CLS → out_proj(tanh(dense(cls))). -pub struct BlasReranker { - w: HashMap>, - tok: Tokenizer, - scratch: Scratch, -} - -impl BlasReranker { - pub fn new() -> Result { - let dir = hf_snapshot("BAAI/bge-reranker-base")?; - Ok(Self { - w: load_weights(&dir)?, - tok: load_tokenizer(&dir)?, - scratch: Scratch::default(), - }) - } -} - -impl Reranker for BlasReranker { - fn rerank(&mut self, query: &str, docs: &[&str]) -> Result> { - let pairs: Vec<(&str, &str)> = docs.iter().map(|d| (query, *d)).collect(); - let encs = self - .tok - .encode_batch(pairs, true) - .map_err(|e| anyhow!("tokenize: {e}"))?; - let (w, scratch) = (&self.w, &mut self.scratch); - let h = RERANK.h; - by_length(encs, RERANK.group, |ids, mask, n, l| { - encode(w, RERANK, ids, mask, n, l, scratch); - let mut cls = vec![0f32; n * h]; - for s in 0..n { - cls[s * h..s * h + h].copy_from_slice(&scratch.hid[(s * l) * h..(s * l) * h + h]); - } - let mut d = vec![0f32; n * h]; - linear( - &cls, - n, - h, - w["classifier.dense.weight"].as_slice(), - w["classifier.dense.bias"].as_slice(), - h, - &mut d, - ); - for v in d.iter_mut() { - *v = v.tanh(); - } - let mut scores = vec![0f32; n]; - linear( - &d, - n, - h, - w["classifier.out_proj.weight"].as_slice(), - w["classifier.out_proj.bias"].as_slice(), - 1, - &mut scores, - ); - scores - }) - } -} From 19a84236ab9ae9fb5e635d0c997a9f8f7a4245c6 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Thu, 12 Nov 2026 23:26:39 +0100 Subject: [PATCH 03/18] feat(bench): time an index's row writing with bench_index An index run writes rows, embeds them, then compacts the memory, and the embedding takes long enough to hide the rest. index::write_rows runs everything but the embedding, and bench_index times it in a scratch memory, over generated sessions or a directory of turns files. It reports the rows written and their rate, the fragments the writes leave for the compaction, and the time the end of the run takes. Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 6 ++ benchmark/README.md | 24 +++++- benchmark/bench_index.rs | 165 +++++++++++++++++++++++++++++++++++++++ src/commands/index.rs | 41 ++++++++++ 4 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 benchmark/bench_index.rs diff --git a/Cargo.toml b/Cargo.toml index 829b94c1..7f28d63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,12 @@ onnx = ["dep:fastembed"] name = "bench_recall" path = "benchmark/bench_recall.rs" +# Time the part of `funes index` that writes rows, without the embedding: +# cargo run --release --example bench_index -- --sessions 2000 2>/dev/null +[[example]] +name = "bench_index" +path = "benchmark/bench_index.rs" + # Time the embedder and the reranker on one backend, blas or, in a macOS build, metal: # cargo run --release --example bench_models -- --backend metal [[example]] diff --git a/benchmark/README.md b/benchmark/README.md index f12fd367..6d85dd5d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,8 +1,9 @@ # Benchmarks -Five runnable examples, each `cargo run --release --example `: +Six runnable examples, each `cargo run --release --example `: - **`bench_recall`** — `recall()` latency, local vs remote, cold vs warm. +- **`bench_index`** — the part of `funes index` that writes rows, without the embedding. - **`bench_models`** — embedding and reranking time on one inference backend. - **`score_models`** — how closely the inference backends reproduce ONNX Runtime's embeddings and reranker scores. @@ -91,6 +92,27 @@ that floor moves with the CPU and the inference backend. What the benchmark meas - For a private dataset, a token must be available (`HF_TOKEN` or the cached login) — the same one recall uses. +## `bench_index` — writing an index's rows + +`bench_index.rs` times the part of `funes index` that writes rows, in a scratch memory. It leaves +the embedding out, since it takes the longest and does not depend on how the rows are written. It +times writing the rows (reading each turns file, scanning it for secrets, chunking it, appending its +rows, recording its state), then the end of the run (the language counts and the compaction). Both +grow with the number of sessions as well as with their size, so a history of many short sessions is +where they show. + +```sh +cargo run --release --example bench_index -- --sessions 2000 2>/dev/null +cargo run --release --example bench_index -- --dir path/to/turns --limit 5000 2>/dev/null +``` + +The corpus is generated, `--sessions` sessions (2000 by default) of `--turns` turns (12) that are +the same on every run, or a directory of turns files given with `--dir`, of which `--limit` keeps +the most recent sessions. The secrets are scanned when TruffleHog is installed, as in +`funes index`. The indexer reports each session and the compaction on stderr. The bench prints the +sessions and chunks written with their time and rate, the fragments the writes left for the +compaction to merge, then the time the end of the run took. + ## `bench_models` — embedding and reranking time `bench_models.rs` times funes's embedder and reranker (gte-reranker-modernbert-base) on one diff --git a/benchmark/bench_index.rs b/benchmark/bench_index.rs new file mode 100644 index 00000000..f892c727 --- /dev/null +++ b/benchmark/bench_index.rs @@ -0,0 +1,165 @@ +//! Time the part of `funes index` that writes rows, in a scratch memory: +//! +//! ```text +//! cargo run --release --example bench_index -- --sessions 2000 2>/dev/null +//! cargo run --release --example bench_index -- --dir path/to/turns --limit 5000 2>/dev/null +//! ``` +//! +//! An index run writes every session's rows, embeds them, then compacts the memory. The embedding +//! takes the longest and does not depend on how the rows are written, so this leaves it out and +//! times the rest: writing the rows (reading each turns file, scanning it for secrets, chunking it, +//! appending its rows, recording its state), then the end of the run (the language counts and the +//! compaction). Both grow with the number of sessions as well as with their size, so a history of +//! many short sessions is where they show. +//! +//! The corpus is generated, `--sessions` sessions of `--turns` turns that are the same on every run, +//! or a directory of turns files given with `--dir`, of which `--limit` keeps the most recent +//! sessions. The secrets are scanned when TruffleHog is installed, as in `funes index`. The indexer +//! reports each session and the compaction on stderr. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use clap::Parser; + +#[derive(Parser)] +struct Args { + /// Sessions to generate. + #[arg(long, default_value_t = 2000, conflicts_with = "dir")] + sessions: usize, + /// Turns per generated session. + #[arg(long, default_value_t = 12, conflicts_with = "dir")] + turns: usize, + /// A directory of turns files to index instead of a generated corpus. + #[arg(long)] + dir: Option, + /// Index only the directory's most recent N sessions. + #[arg(long, requires = "dir")] + limit: Option, +} + +/// The words a generated turn is drawn from. +const WORDS: [&str; 48] = [ + "the", + "index", + "writes", + "rows", + "for", + "each", + "session", + "and", + "then", + "embeds", + "them", + "while", + "recall", + "reads", + "chunks", + "back", + "from", + "a", + "memory", + "on", + "disk", + "with", + "its", + "turns", + "in", + "order", + "so", + "an", + "agent", + "can", + "find", + "what", + "was", + "decided", + "before", + "about", + "this", + "repository", + "build", + "test", + "run", + "error", + "fix", + "plan", + "review", + "answer", + "question", + "user", +]; + +/// An xorshift generator with a fixed seed, so a generated corpus is the same on every run. +struct Rng(u64); + +impl Rng { + fn below(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % n as u64) as usize + } +} + +/// Write `sessions` turns files of `turns` turns each into `dir`, a turn of 40 to 400 words. +fn generate(dir: &Path, sessions: usize, turns: usize) -> Result<()> { + std::fs::create_dir_all(dir)?; + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for s in 0..sessions { + let session_id = format!("bench-{s:06}"); + let mut file = String::new(); + for seq in 0..turns { + let words: Vec<&str> = (0..40 + rng.below(361)) + .map(|_| WORDS[rng.below(WORDS.len())]) + .collect(); + let turn = serde_json::json!({ + "session_id": session_id, + "turn_uuid": format!("t{seq}"), + "seq": seq, + "ts": "2026-01-01T00:00:00Z", + "role": if seq % 2 == 0 { "user" } else { "assistant" }, + "harness": "bench", + "blocks": [{"block_type": "text", "text": words.join(" ")}], + }); + file.push_str(&turn.to_string()); + file.push('\n'); + } + std::fs::write(dir.join(format!("{session_id}.funes.jsonl")), file)?; + } + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + let scratch = tempfile::tempdir()?; + let (dir, corpus) = match &args.dir { + Some(dir) => { + let kept = args.limit.map(|n| format!(", its {n} most recent sessions")); + (dir.clone(), format!("{}{}", dir.display(), kept.unwrap_or_default())) + } + None => { + let dir = scratch.path().join("turns"); + generate(&dir, args.sessions, args.turns)?; + ( + dir, + format!("{} generated sessions of {} turns", args.sessions, args.turns), + ) + } + }; + // A scratch home: the run builds a memory of its own and leaves yours alone. + std::env::set_var("FUNES_HOME", scratch.path().join("funes")); + let run = funes::commands::index::write_rows(&dir, args.limit).await?; + let write = run.write.as_secs_f64(); + println!("\ncorpus: {corpus}"); + println!( + "rows: {} sessions, {} chunks in {write:.1} s, {:.1} sessions/s", + run.sessions, + run.chunks, + run.sessions as f64 / write + ); + println!("left: {} fragments to merge", run.fragments); + println!("end of run: {:.1} s", run.finalize.as_secs_f64()); + Ok(()) +} diff --git a/src/commands/index.rs b/src/commands/index.rs index 8d5d413a..6fffd466 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -1254,6 +1254,47 @@ pub async fn run_index(path: &Path, no_thinking: bool, max_sessions: Option) -> Result { + let mut indexer = Indexer::open(vec![source::open(path, max_sessions)?], false).await?; + let total = indexer.unit_count(); + let all: Vec = (0..total).collect(); + let t0 = Instant::now(); + let mut done = 0; + while done < total { + done += indexer.write_units(&all[done..], done, total).await?; + } + let write = t0.elapsed(); + let fragments = match &indexer.ds { + Some(ds) => dataset::fragments_to_compact(ds).await?, + None => 0, + }; + let (sessions, chunks) = (indexer.n_sessions, indexer.n_chunks); + indexer.work_remaining = true; + let t1 = Instant::now(); + indexer.finalize().await?; + Ok(RowsWritten { + sessions, + chunks, + fragments, + write, + finalize: t1.elapsed(), + }) +} + /// A first interactive index estimated at ≥ this many seconds prompts before continuing. const FIRST_INDEX_PROMPT_SECS: u64 = 120; From 6f0a1d4f197086135aefb10a389fcb3614540e75 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 08:52:58 +0100 Subject: [PATCH 04/18] perf(index): persist the index state once per batch record rewrote state.json and index-coverage.json after every unit, and each write holds every unit the run has seen, so a run over many small sessions paid a cost that grew with the square of their count. The state is now persisted once per batch, after a record, and the batch's spool files are drained after that write. A run that fails mid-batch re-reads that batch on its next run, where the stored chunk ids keep its rows from being written twice. bench_index, rows written before and after: - 5,000 day-zero sessions: 181.6 s, then 167.6 s - 2,000 generated sessions: 64.9 s, then 64.1 s Co-Authored-By: Claude Opus 5.5 --- src/commands/index.rs | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/commands/index.rs b/src/commands/index.rs index 6fffd466..6a883a35 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -497,6 +497,11 @@ struct Indexer { /// Units (by index) whose read failed this run — reported once. Signed units also keep their /// refusal in state until their content or this build changes. rejected: HashSet, + /// A unit's state was recorded since the last [`Indexer::persist`]. + recorded: bool, + /// The spool files this batch indexed, as key, signature and progress label: drained once + /// [`Indexer::persist`] has recorded them. + drained: Vec<(String, String, String)>, } /// Enumerate every source's units (each source orders its own — recency-desc), tagged with the @@ -611,6 +616,8 @@ impl Indexer { n_embedded: 0, languages, rejected: HashSet::new(), + recorded: false, + drained: Vec::new(), }) } @@ -637,9 +644,18 @@ impl Indexer { .collect() } - /// Record a unit's state and persist it, so an interrupted run resumes where it stopped. - fn record(&mut self, key: &str, state: UnitState) -> Result<()> { + /// Record a unit's state, which [`Indexer::persist`] writes with the rest of its batch. + fn record(&mut self, key: &str, state: UnitState) { self.state.insert(key.to_string(), state); + self.recorded = true; + } + + /// Persist the recorded states, so an interrupted run resumes where it stopped. Each write holds + /// every unit's state, so it runs once a batch rather than once a unit, and only after a record. + fn persist(&mut self) -> Result<()> { + if !std::mem::take(&mut self.recorded) { + return Ok(()); + } std::fs::write(&self.state_path, serde_json::to_string_pretty(&self.state)?)?; write_index_coverage(&self.coverage_path, &self.sources, &self.units, &self.state) } @@ -676,7 +692,7 @@ impl Indexer { level: None, refused: Some(VERSION.to_string()), }, - )?; + ); } Ok(None) } @@ -697,9 +713,10 @@ impl Indexer { Ok(read) } - /// Write all tiers of unit `i` from sanitized `turns`, keeping only new chunks in one append. - /// Record that its rows are in the memory before draining its spool file; the memory itself - /// carries any embeddings still owed. A bulk source stays one fragment per unit. + /// Write all tiers of unit `i` from sanitized `turns`, keeping only new chunks in one append, + /// and record that its rows are in the memory. Its spool file is drained once + /// [`Indexer::write_units`] has persisted that record, and the memory itself carries any + /// embeddings still owed. A bulk source stays one fragment per unit. async fn write_unit(&mut self, i: usize, turns: &[traces::Turn], progress: &str) -> Result { let (key, sig) = { let unit = &self.units[i].1; @@ -751,11 +768,9 @@ impl Indexer { level: Some(Tier::ToolResult), refused: None, }, - )?; + ); if drains(&key, Tier::ToolResult, self.include_thinking) { - if let Err(e) = drain(&key, sig) { - eprintln!("{progress} {key} — indexed, but the spool copy stayed: {e:#}"); - } + self.drained.push((key, sig.clone(), progress.to_string())); } } self.n_sessions += sessions; @@ -774,6 +789,13 @@ impl Indexer { self.write_unit(units[n], &turns, &label(n)).await?; } } + // A spool file is drained only once the state says its unit is indexed. + self.persist()?; + for (key, sig, progress) in std::mem::take(&mut self.drained) { + if let Err(e) = drain(&key, &sig) { + eprintln!("{progress} {key} — indexed, but the spool copy stayed: {e:#}"); + } + } Ok(consumed) } From df922a67642502ecd0774b3693bd3032fc5b5bdd Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 11:08:46 +0100 Subject: [PATCH 05/18] feat(push): redact the secrets a push finds instead of holding them back The push gate held back every block that still held a secret, and its rows waited for funes scrub. It now redacts each such block as scrub does: the secret excised from the block's text, the text re-split by the same chunker, the result scanned again and re-embedded. The pieces are published under their rows' ids, so the remote holds the ids the local memory does, and the local memory keeps its rows as they are. A block that can't be redacted in place is still held back whole: a secret in an encoded region, a redaction that still scans dirty, or one that re-splits into a different number of pieces. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- docs/automation.md | 7 +- docs/push.md | 10 +- src/commands/push.rs | 269 +++++++++++++++++++++++++++++++++---------- 4 files changed, 219 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 082db381..c9c352c0 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ funes recall "..." --memory /funes-memory # read any remote memory f That second form is how you read **someone else's** published memory on a topic, without touching your own setup. Publishing is guarded: funes redacts credentials at index time, and a separate, -always-on gate refuses to push any chunk that still contains a secret. And because a published memory -is just a dataset, you can try recall **right now**, before indexing anything of your own: +always-on gate redacts any left in what it publishes, holding back what it can't redact. And because +a published memory is just a dataset, you can try recall **right now**, before indexing anything of +your own: ```bash funes recall "why is funes append-only" --memory huggingface/funes-memory diff --git a/docs/automation.md b/docs/automation.md index f7eb3a7e..85c02fbb 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -78,9 +78,10 @@ validates a producer's output without writing anything, so run it before wiring memory compete for its head on the Hub, and the one that loses retries. The one case that can leave rows out is a first publish to an empty memory, which has no head to check against. Setting up one host before the others avoids it. -- **Secrets held back.** funes redacts credentials at index time; on push, a separate always-on gate - withholds any chunk that still contains one — the clean rows publish, and the push exits non-zero - (code `2`) only if that leaves nothing to publish. Run `funes scrub`, then the next push includes it. +- **Secrets redacted on the way out.** funes redacts credentials at index time, and on push a + separate always-on gate redacts any left in what it publishes. A block it can't redact in place is + held back, the clean rows publish, and the push exits non-zero (code `2`) only if that leaves + nothing to publish. Run `funes scrub`, then the next push includes it. - **The card rides along.** A push to a memory at the repo root creates the repo's dataset card (tagged `funes`) and keeps its stats fresh — in the same commit as the data. A hand-written card is never touched. diff --git a/docs/push.md b/docs/push.md index e9394175..9f7b6072 100644 --- a/docs/push.md +++ b/docs/push.md @@ -56,9 +56,13 @@ has not crossed a publication boundary. It prints a warning that index-time reda Push is the hard boundary. A separate, **always-on, fail-closed gate** requires TruffleHog and scans the rows about to leave the machine. It reconstructs complete content blocks before scanning, so a -secret split across chunks cannot evade detection. If any chunk of a block contains a secret, every -chunk of that block is held back; unrelated clean rows still publish with a warning. Only when that -leaves *nothing* to publish does push exit non-zero (code `2`): +secret split across chunks cannot evade detection. A block that contains a secret is redacted the +way [`funes scrub`](#what-funes-scrub-changes) does it, and its redacted chunks are published in +place of the stored ones, which the local memory keeps as they are. A block that can't be redacted +in place is held back whole: a secret found in an encoded region, a redaction that still scans dirty, +or one that re-chunks into a different number of chunks than the block has. Unrelated clean rows +still publish with a warning. Only when that leaves *nothing* to publish does push exit non-zero +(code `2`): ```console $ funes push /funes-memory diff --git a/src/commands/push.rs b/src/commands/push.rs index 9720e418..be204273 100644 --- a/src/commands/push.rs +++ b/src/commands/push.rs @@ -2,9 +2,9 @@ //! //! Streamed, never a full mirror. "What's already there" is the memory's own chunk ids, so the //! delta is `embedded_local_ids − remote_ids`. `push` is orchestration: -//! it computes the delta, holds back any row that still contains a secret (redaction happens at -//! index time; this is the egress backstop — the rows wait for `funes scrub`), and drives the HF -//! write operations in [`crate::memory::remote`], which own the atomic, parent-commit-guarded commits. +//! it computes the delta, redacts the secrets still in the rows it ships (holding back the blocks it +//! can't redact in place, which wait for `funes scrub`), and drives the HF write operations in +//! [`crate::memory::remote`], which own the atomic, parent-commit-guarded commits. //! //! - **First publish:** build the dataset locally (data + FTS/IVF indexes) and upload every file in //! one commit. @@ -22,6 +22,7 @@ use super::compact; use crate::hub; +use crate::inference::{self, embed_batched, EmbeddingModel}; use crate::memory::card::{self, CardAction, CardCtx}; use crate::memory::dataset::{self, RecordBatch}; use crate::memory::lock; @@ -86,8 +87,8 @@ async fn held_among(local: &Dataset, pending: &HashSet) -> Option 0).then_some(skipped) + let held = gate(&rows).ok()?.held; + (held.rows > 0).then_some(held) } /// Every embedded row of the local memory, whole — a publish ships each row as it stands. @@ -401,10 +402,11 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: return Ok(format!("{}: up to date ({} chunks)\n{note}", target.label(), remote_ids.len()).into()); } - // 4. Rows, then hold back any that still contain a secret. Re-stamp each batch with the local + // 4. Rows, then redact the secrets they still hold. Re-stamp each batch with the local // dataset's schema so its metadata (the embedding-model id) rides along — scan-result batches // drop it, and on first publish that schema is what the new dataset persists. let schema = dataset::schema_of(&local); + let model = dataset::embedding_model(&local)?; // Only a first publish with nothing named ships the whole memory; anything else, `to_push` // is the answer — including a first publish of a selection. let rows = if first_publish && sessions.is_empty() { @@ -414,11 +416,23 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: }; let batches = dataset::restamped(&schema, rows)?; - // Drop any row whose text still holds a secret — hold it back from the Hub rather than block the - // whole push. `funes scrub` redacts it in the local memory; the next push then ships it. + // Redact the secrets the rows still hold, and hold back the blocks that can't be redacted in + // place rather than block the whole push: `funes scrub` redacts or drops them in the local + // memory, and the next push ships what it leaves. let n_scanning: usize = batches.iter().map(|b| b.num_rows()).sum(); eprintln!("scanning {n_scanning} chunk(s) for secrets…"); - let (batches, skipped) = drop_secret_rows(batches)?; + let gated = gate(&batches)?; + let batches = gated_rows(batches, &gated, model, |texts| { + eprintln!("re-embedding {} redacted chunk(s)…", texts.len()); + let mut embedder = inference::embedder(model)?; + embed_batched(embedder.as_mut(), texts, |_| {}) + })?; + let batches = dataset::restamped(&schema, batches)?; + let Gated { + held: skipped, + redacted, + .. + } = gated; let n_chunks: usize = batches.iter().map(|b| b.num_rows()).sum(); if n_chunks == 0 && skipped.rows > 0 { // Everything was held back: nothing reached the Hub. Mark `blocked` so the CLI exits non-zero @@ -441,7 +455,6 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: // (and two stores sharing a repo would fight over one card's stats) — under a prefix it's // the owner's. The chunk count is the post-push total — best-effort: rows another writer // lands concurrently are missed until the next push refreshes the card. - let model = dataset::embedding_model(&local)?; let date = Utc::now().format("%Y-%m-%d").to_string(); let ctx = CardCtx { repo: &w.repo_id, @@ -485,8 +498,9 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: n => format!("pushed {n} chunks"), }; return Ok(format!( - "{}: {pushed} (commit {oid})\n{card_note}{}", + "{}: {pushed} (commit {oid})\n{card_note}{}{}", target.label(), + redacted.note(), skipped.warning() ) .into()); @@ -539,6 +553,7 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: eprintln!("compacting the remote…"); out.push_str(&compact_auto(&w).await); } + out.push_str(&redacted.note()); out.push_str(&skipped.warning()); Ok(out.into()) } @@ -565,6 +580,7 @@ fn card_file(remote_readme: Result>, ctx: &CardCtx) -> (Option) -> Result<(Vec, Skipped)> { +/// Blocks a push redacted secrets from before publishing them. +#[derive(Default)] +struct Redacted { + blocks: usize, + /// Detectors that fired, e.g. `PrivateKey×1, AWS×2`. + summary: String, +} + +impl Redacted { + /// A line for the push output, or empty if nothing was redacted. + fn note(&self) -> String { + if self.blocks == 0 { + return String::new(); + } + format!( + " redacted secrets in {} block(s) before publishing ({})\n", + self.blocks, self.summary + ) + } +} + +/// What the secret gate makes of the rows a push would upload. +struct Gated { + /// Per row, across the batches in order: its block held a secret, so the row is replaced or held + /// back. + dirty: Vec, + /// The redacted blocks' pieces, each carrying the id of the row it replaces. + replacements: Vec, + redacted: Redacted, + held: Skipped, +} + +/// Scan the to-push `batches` for secrets, a whole block at a time: a block's chunks are +/// reconstructed into their contiguous text (so a secret `split` cut across chunks is whole and +/// detectable), and every block is scanned in one pass. A block that holds a secret is redacted as +/// `funes scrub` does it: the secret excised from its text, the text re-split by the same chunker, +/// the result scanned again. Its pieces replace its rows only if they carry the same ids, the +/// block's coordinates, so the remote holds the ids the local memory does. Otherwise the whole +/// block is held back, its other chunks carrying the rest of the secret: a secret the scanner +/// decoded from an encoded region, a redaction that still scans dirty, one re-split into another +/// number of pieces. Fail-closed on the scanner — a push must scan before it uploads. +fn gate(batches: &[RecordBatch]) -> Result { + let mut gated = Gated { + dirty: Vec::new(), + replacements: Vec::new(), + redacted: Redacted::default(), + held: Skipped::default(), + }; // Nothing to scan needs no scanner. if batches.iter().all(|b| b.num_rows() == 0) { - return Ok(( - batches, - Skipped { - rows: 0, - summary: String::new(), - }, - )); + return Ok(gated); } let scanner = scan::Trufflehog::find()?; // Row order across batches matches `chunks_from_batches`, so a chunk's index is its global row. - let chunks = dataset::chunks(&batches); + let chunks = dataset::chunks(batches); let blocks = chunk::reconstruct_blocks(&chunks); let texts: Vec<&str> = blocks.iter().map(|(_, text)| text.as_str()).collect(); let found = scan::scan_blocks(&texts, &scanner)?; + gated.dirty = vec![false; chunks.len()]; - let mut dirty = vec![false; chunks.len()]; - let mut detectors: Vec = Vec::new(); - for ((idxs, _), findings) in blocks.iter().zip(&found) { + // One tally per (block, distinct detector), so a summary reads "PrivateKey×". + let (mut redacted, mut held) = (Vec::new(), Vec::new()); + let mut candidates = Vec::new(); + for ((idxs, text), findings) in blocks.iter().zip(&found) { if findings.is_empty() { continue; } for &i in idxs { - dirty[i] = true; + gated.dirty[i] = true; + } + let r = scan::excise(text, findings); + let pieces = chunk::resplit(&chunks[idxs[0]], &r.text); + let same_ids = pieces.iter().map(|p| &p.id).eq(idxs.iter().map(|&i| &chunks[i].id)); + if r.fully_redacted && same_ids { + candidates.push((idxs, findings, r.text, pieces)); + } else { + gated.held.rows += idxs.len(); + held.extend(scan::detectors(findings)); } - // One tally per (block, distinct detector), so the warning reads "PrivateKey×". - detectors.extend(scan::detectors(findings)); } - let dropped = dirty.iter().filter(|&&d| d).count(); - if dropped == 0 { - return Ok(( - batches, - Skipped { - rows: 0, - summary: String::new(), - }, - )); + // Excising one match can expose another (removing one from a long base64 run re-aligns the + // window the next is found in), so a redaction stands only if it scans clean. + if !candidates.is_empty() { + let rescan: Vec<&str> = candidates.iter().map(|(_, _, text, _)| text.as_str()).collect(); + let left = scan::scan_blocks(&rescan, &scanner)?; + for ((idxs, findings, _, pieces), left) in candidates.into_iter().zip(&left) { + if left.is_empty() { + gated.replacements.extend(pieces); + gated.redacted.blocks += 1; + redacted.extend(scan::detectors(findings)); + } else { + gated.held.rows += idxs.len(); + held.extend(scan::detectors(findings)); + } + } } + gated.redacted.summary = scan::summary(redacted.iter().map(String::as_str)); + gated.held.summary = scan::summary(held.iter().map(String::as_str)); + Ok(gated) +} - // Drop the dirty rows batch by batch, mapping each global row index back via a running offset. - let mut clean = Vec::with_capacity(batches.len()); +/// The rows a push uploads once gated: every row outside a dirty block as it stands, then the +/// redacted blocks' pieces under their rows' ids, embedded by `embed`, which only runs when a block +/// was redacted. The rows of a held-back block stay behind. +fn gated_rows( + batches: Vec, + gated: &Gated, + model: EmbeddingModel, + embed: impl FnOnce(&[&str]) -> Result>>, +) -> Result> { + if !gated.dirty.contains(&true) { + return Ok(batches); + } + // Keep the rows of clean blocks batch by batch, mapping each global row index back via a running + // offset. + let mut out = Vec::with_capacity(batches.len() + 1); let mut base = 0usize; for b in &batches { - clean.push(dataset::keep_rows(b, |i| !dirty[base + i])?); + out.push(dataset::keep_rows(b, |i| !gated.dirty[base + i])?); base += b.num_rows(); } - let summary = scan::summary(detectors.iter().map(String::as_str)); - Ok((clean, Skipped { rows: dropped, summary })) + if !gated.replacements.is_empty() { + let texts: Vec<&str> = gated.replacements.iter().map(|c| c.text.as_str()).collect(); + let vectors = embed(&texts)?; + out.push(dataset::build_batch(model, &gated.replacements, Some(&vectors))?); + } + Ok(out) } /// Best-effort compaction during a normal push: one attempt, never retried. The data is already @@ -844,12 +930,54 @@ mod tests { ) } + /// Zero vectors in place of an embedder, for the rows a gate redacted. + fn zero_vectors(texts: &[&str]) -> Result>> { + Ok(vec![vec![0.0f32; dataset::DIM as usize]; texts.len()]) + } + #[test] - fn drop_secret_rows_holds_back_an_escaped_key() { + fn a_redacted_block_ships_under_its_rows_ids() { + if scan::Trufflehog::find().is_err() { + eprintln!("skip: trufflehog not found"); + return; + } + let Some(key) = keygen(&["-t", "ed25519"]) else { + eprintln!("skip: ssh-keygen unavailable"); + return; + }; + let (b, chunks) = batch(&[turn(0, "clean note"), turn(1, &format!("deploy key: {key}"))]); + assert_eq!(chunks.len(), 2, "precondition: one chunk per turn"); + let ids = dataset::ids_in(std::slice::from_ref(&b)); + + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!( + gated.held.rows, 0, + "a key whose redaction keeps the block's pieces is redacted, not held" + ); + assert_eq!(gated.redacted.blocks, 1); + assert!( + gated.redacted.summary.contains("PrivateKey"), + "summary: {}", + gated.redacted.summary + ); + + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + assert_eq!(dataset::ids_in(&shipped), ids, "the redacted row keeps its id"); + let row = dataset::chunks(&shipped) + .into_iter() + .find(|c| c.id == chunks[1].id) + .unwrap(); + assert!(row.text.contains("[REDACTED:PrivateKey]"), "text: {}", row.text); + let body = key.lines().nth(1).unwrap(); + assert!(!row.text.contains(body), "no line of the key ships"); + } + + #[test] + fn an_escaped_key_is_redacted_not_pushed() { // The exact shape that leaked: a key stored with escaped `\n` (literal backslash-n), as a // JSON-encoded transcript or a logged blob would hold it. trufflehog still detects it, but - // its canonical `raw` (real newlines) is not a substring of the stored bytes — value - // matching missed it and pushed it. Line-based location must hold it back. + // its canonical `raw` (real newlines) is not a substring of the stored bytes, so value + // matching missed it and pushed it. Its escaped form must be found and redacted. if scan::Trufflehog::find().is_err() { eprintln!("skip: trufflehog not found"); return; @@ -862,14 +990,14 @@ mod tests { assert!(!escaped.contains('\n'), "precondition: no real newlines remain"); let (b, _) = batch(&[turn(0, "clean note"), turn(1, &format!("deploy key: {escaped}"))]); - let (clean, skipped) = drop_secret_rows(vec![b]).unwrap(); - assert_eq!(skipped.rows, 1, "the escaped key must be held back, not pushed"); - assert!(skipped.summary.contains("PrivateKey"), "summary: {}", skipped.summary); - assert_eq!( - clean.iter().map(|b| b.num_rows()).sum::(), - 1, - "the clean row stays" - ); + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!(gated.held.rows, 0, "the escaped key is redacted, not held"); + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + assert_eq!(shipped.iter().map(|b| b.num_rows()).sum::(), 2, "both rows ship"); + let body = key.lines().nth(1).unwrap(); + for row in dataset::chunks(&shipped) { + assert!(!row.text.contains(body), "no line of the key ships: {}", row.text); + } } /// A two-session local dataset for the selection tests. @@ -960,12 +1088,15 @@ mod tests { eprintln!("skip: trufflehog not found"); return; } - let Some(key) = keygen(&["-t", "ed25519"]) else { + // A key that is the whole of a block spanning several chunks: its redaction re-splits into + // fewer pieces, so a push holds the block back rather than redact it in place. + let Some(key) = keygen(&["-t", "rsa", "-b", "4096"]) else { eprintln!("skip: ssh-keygen unavailable"); return; }; let dir = tempfile::tempdir().unwrap(); let (b, _) = batch(&[turn_sess("clean", 0, "notes on parsers"), turn_sess("dirty", 1, &key)]); + assert!(b.num_rows() > 2, "the secret must span multiple chunks"); let uri = dataset::table_uri(dir.path()); let schema = b.schema(); let reader = RecordBatchIterator::new(vec![Ok(b)], schema); @@ -979,7 +1110,7 @@ mod tests { let held = held_among(&ds, &all) .await .expect("the dirty pending block should be reported"); - assert!(held.rows >= 1); + assert_eq!(held.rows, by_session["dirty"].len()); assert!(held.summary.contains("PrivateKey"), "summary: {}", held.summary); let clean_only: HashSet = by_session["clean"].iter().cloned().collect(); @@ -1023,14 +1154,24 @@ mod tests { let all: HashSet = by_session.values().flatten().cloned().collect(); let rows = rows_with_ids(&ds, &all).await.unwrap(); - let (clean, held) = drop_secret_rows(rows).unwrap(); - assert_eq!(held.rows, by_session["dirty"].len(), "every secret split is held"); - assert!(held.summary.contains("PrivateKey"), "summary: {}", held.summary); + let gated = gate(&rows).unwrap(); + // The key is the block's whole text, so its redaction re-splits into fewer pieces than the + // rows it would replace: held back whole. + assert_eq!(gated.held.rows, by_session["dirty"].len(), "every secret split is held"); + assert!( + gated.held.summary.contains("PrivateKey"), + "summary: {}", + gated.held.summary + ); + let clean = gated_rows(rows, &gated, dataset::model::ENGLISH, |_| { + unreachable!("nothing redacted") + }) + .unwrap(); let clean_only: HashSet = by_session["clean"].iter().cloned().collect(); assert_eq!(dataset::ids_in(&clean), clean_only, "unrelated clean rows survive"); let rows = rows_with_ids(&ds, &clean_only).await.unwrap(); - assert_eq!(drop_secret_rows(rows).unwrap().1.rows, 0, "clean rows are not held"); + assert_eq!(gate(&rows).unwrap().held.rows, 0, "clean rows are not held"); } #[tokio::test] From d018c196f96f5ef2932922d9f3a5d557ca1e7ff9 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 11:46:13 +0100 Subject: [PATCH 06/18] perf(index): leave secret redaction to push Indexing spawned TruffleHog for every batch of 32 sessions, and so for every agent turn the hook indexed: about 1 s of CPU and 160 MB each, for a few kilobytes of text. Push redacts what it publishes now, so indexing stores sessions as written and only elides data URIs. The local memory is no publication boundary, and push and scrub still refuse to run without a scanner. bench_index, the two builds alternated, rows written before and after: - 5,000 day-zero sessions: 179.4 s, then 20.7 s (294.6 s, then 34.6 s in a busier first round) - 2,000 generated sessions: 65.4 s, then 4.9 s The end of the run, merging and indexing, takes the same time in both. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 +- docs/automation.md | 4 +- docs/configuration.md | 2 +- docs/funes-jsonl.md | 5 +- docs/push.md | 5 +- src/chunk.rs | 5 +- src/commands/index.rs | 233 ++++-------------------- src/main.rs | 6 +- tests/{scan_batch.rs => index_batch.rs} | 28 ++- tests/redact_on_index.rs | 72 -------- tests/scrub.rs | 10 +- tests/scrub_drops_all.rs | 6 +- tests/scrub_redacts_escaped_key.rs | 6 +- 13 files changed, 68 insertions(+), 321 deletions(-) rename tests/{scan_batch.rs => index_batch.rs} (79%) delete mode 100644 tests/redact_on_index.rs diff --git a/README.md b/README.md index c9c352c0..27deb326 100644 --- a/README.md +++ b/README.md @@ -106,10 +106,9 @@ funes recall "..." --memory /funes-memory # read any remote memory f ``` That second form is how you read **someone else's** published memory on a topic, without touching -your own setup. Publishing is guarded: funes redacts credentials at index time, and a separate, -always-on gate redacts any left in what it publishes, holding back what it can't redact. And because -a published memory is just a dataset, you can try recall **right now**, before indexing anything of -your own: +your own setup. Publishing is guarded: an always-on gate redacts credentials from what it publishes, +and holds back what it can't redact. And because a published memory is just a dataset, you can try +recall **right now**, before indexing anything of your own: ```bash funes recall "why is funes append-only" --memory huggingface/funes-memory diff --git a/docs/automation.md b/docs/automation.md index 85c02fbb..53bb5d8e 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -78,8 +78,8 @@ validates a producer's output without writing anything, so run it before wiring memory compete for its head on the Hub, and the one that loses retries. The one case that can leave rows out is a first publish to an empty memory, which has no head to check against. Setting up one host before the others avoids it. -- **Secrets redacted on the way out.** funes redacts credentials at index time, and on push a - separate always-on gate redacts any left in what it publishes. A block it can't redact in place is +- **Secrets redacted on the way out.** The local memory keeps sessions as written, and on push an + always-on gate redacts credentials from what it publishes. A block it can't redact in place is held back, the clean rows publish, and the push exits non-zero (code `2`) only if that leaves nothing to publish. Run `funes scrub`, then the next push includes it. - **The card rides along.** A push to a memory at the repo root creates the repo's dataset card diff --git a/docs/configuration.md b/docs/configuration.md index 94b2b9b9..d6ddef1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,7 +88,7 @@ file-grained cache design and cold-versus-warm behavior. | `FUNES_BIN` | Binary path recorded in supported MCP registrations and used by the pi bridge. Hook workers instead find `funes` on `PATH` or in common install directories. | | `FUNES_MEMORY` | Per-run memory override understood by the pi extension; otherwise its copy of the host's binding is used. | | `FUNES_INTEGRATIONS` | Directory `funes add` installs an agent integration from, instead of the published one. funes can't vouch for those files, so it asks you to confirm before running them — once, until they change. | -| `FUNES_TRUFFLEHOG` | Explicit TruffleHog binary for secret scanning. Index-time redaction is best-effort; push and scrub scanning fail closed. | +| `FUNES_TRUFFLEHOG` | Explicit TruffleHog binary for secret scanning. Push and scrub refuse to run without one. | | `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN`, `HUGGINGFACE_TOKEN` | Hugging Face authentication, in the precedence shown above. | | `HF_TOKEN_PATH` | The token file, when it is not `$HF_HOME/token`. | | `HF_HUB_DISABLE_IMPLICIT_TOKEN` | Set to anything: no token is used, from the variables or a file. | diff --git a/docs/funes-jsonl.md b/docs/funes-jsonl.md index 34e66d42..e4597c26 100644 --- a/docs/funes-jsonl.md +++ b/docs/funes-jsonl.md @@ -125,8 +125,9 @@ Run it before you publish a producer. Behaviour, not contract — it may evolve; the identity rule above will not. - **Elides** inline base64 `data:` payloads to `[elided]`. -- **Redacts** secrets before chunking, best-effort; whatever slips through is caught by the fail-closed - gate on `push`, so a leaked token never reaches a published memory. +- **Redacts** secrets only on `push`: the local memory keeps them as written, and the fail-closed + gate on `push` redacts them from what it publishes, so a leaked token never reaches a published + memory. - **Renders** each block to the text that is embedded and full-text indexed: `text` and `thinking` as-is; `tool_use` as `[tool_use ] ` (`[tool_use None] ` without a name); `tool_result` as `[tool_result ] ` (`[tool_result] ` without a name). diff --git a/docs/push.md b/docs/push.md index 9f7b6072..f497cc4a 100644 --- a/docs/push.md +++ b/docs/push.md @@ -50,9 +50,8 @@ run this at session boundaries automatically; see [automation.md](automation.md) ## Keeping secrets out: the gate and `funes scrub` -When TruffleHog is available, indexing redacts detected credentials before storing a session. That -first pass is best-effort: local indexing still works without the scanner because the local memory -has not crossed a publication boundary. It prints a warning that index-time redaction is disabled. +Indexing stores sessions as written, credentials included: the local memory is not a publication +boundary, so indexing doesn't scan it. Push is the hard boundary. A separate, **always-on, fail-closed gate** requires TruffleHog and scans the rows about to leave the machine. It reconstructs complete content blocks before scanning, so a diff --git a/src/chunk.rs b/src/chunk.rs index 2d0b3f89..8fa3c17c 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -268,9 +268,8 @@ pub(crate) fn resplit(template: &Chunk, text: &str) -> Vec { } /// Whether an index (with `include_thinking`) stores a block of this type: it isn't a dropped -/// thinking block. Shared with redaction, so the blocks scanned for secrets are exactly the blocks -/// stored. -pub fn block_selected(block_type: &str, include_thinking: bool) -> bool { +/// thinking block. +fn block_selected(block_type: &str, include_thinking: bool) -> bool { block_type != "thinking" || include_thinking } diff --git a/src/commands/index.rs b/src/commands/index.rs index 6a883a35..da04ed03 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -15,7 +15,6 @@ use crate::inference::{self, embed_batched, Embedder}; use crate::memory::dataset::{self, build_batch, model}; use crate::memory::lock; use crate::memory::Memory; -use crate::scan; use crate::traces::spool; use crate::traces::{self, repo, source}; use crate::ui::prompt::confirm; @@ -162,10 +161,9 @@ pub async fn ensure_local_memory(model: inference::EmbeddingModel) -> Result<()> } } -/// Elide every block's inline base64 `data:` URI payloads, before [`redact_units`] scans them: a -/// screenshot's entropy trips detectors on values that are not secrets, and excising one of those -/// plants a marker mid-payload that strands the rest of it in the store. Runs whether or not a -/// scanner is installed — the payload is unrecallable either way. +/// Elide every block's inline base64 `data:` URI payloads before they are stored: a payload is +/// unrecallable, and a screenshot's entropy trips the push gate's detectors on values that are not +/// secrets. fn elide_turns(turns: &mut [traces::Turn]) { for b in turns.iter_mut().flat_map(|t| t.blocks.iter_mut()) { if let Cow::Owned(elided) = chunk::elide_data_uris(&b.text) { @@ -174,82 +172,16 @@ fn elide_turns(turns: &mut [traces::Turn]) { } } -/// Redact secrets from `units`' turns *before* chunking — so a long key that chunking would split -/// across pieces is whole when scanned, and never reaches the embedding, the local memory, or (via -/// push) the Hub. One scanner run covers every unit given: the spawn costs ~1 s, the scan itself -/// milliseconds. Scans exactly the blocks the memory stores ([`chunk::block_selected`]). -/// Best-effort: removes a secret whose value byte-matches the stored text (the common case, real -/// newlines); anything that resists is caught downstream by the fail-closed push gate. Reports to -/// stderr what it removed, per unit. -fn redact_units( - units: &mut [&mut [traces::Turn]], - scanner: &dyn scan::SecretScanner, - include_thinking: bool, -) -> Result<()> { - let n_units = units.len(); - let mut blocks: Vec<(usize, &mut traces::Block)> = Vec::new(); - for (u, turns) in units.iter_mut().enumerate() { - for b in turns.iter_mut().flat_map(|t| t.blocks.iter_mut()) { - if chunk::block_selected(&b.block_type, include_thinking) { - blocks.push((u, b)); - } - } - } - if blocks.is_empty() { - return Ok(()); - } - let per_block = { - let texts: Vec<&str> = blocks.iter().map(|(_, b)| b.text.as_str()).collect(); - scan::scan_blocks(&texts, scanner)? - }; - let mut removed: Vec> = (0..n_units).map(|_| Vec::new()).collect(); - for ((u, b), findings) in blocks.into_iter().zip(&per_block) { - let r = scan::excise(&b.text, findings); - removed[u].extend(r.removed_detectors); - b.text = r.text; - } - for (turns, removed) in units.iter().zip(removed) { - if removed.is_empty() { - continue; - } - let sid = turns.first().map(|t| t.session_id.as_str()).unwrap_or("?"); - eprintln!( - " redacted {} secret(s) in {sid}: {}", - removed.len(), - scan::summary(removed.iter().map(String::as_str)) - ); - } - Ok(()) -} - -/// Make a batch of units' turns what the memory stores — data URIs elided and, with a `scanner`, -/// secrets redacted, the batch in one scanner run — before any of them is chunked, since both -/// change the text a block splits into. -fn sanitize_units( - units: &mut [&mut [traces::Turn]], - include_thinking: bool, - scanner: Option<&impl scan::SecretScanner>, -) -> Result<()> { - for turns in units.iter_mut() { - elide_turns(turns); - } - if let Some(scanner) = scanner { - redact_units(units, scanner, include_thinking)?; - } - Ok(()) -} - -/// Units a batch consumes: those read are scanned for secrets in one scanner spawn (~1 s) rather -/// than one per unit, so a directory of hundreds of session files is scanned in seconds, not -/// minutes. -const SCAN_BATCH: usize = 32; +/// Units a batch consumes. A run reads, writes and records its units a batch at a time, and a +/// budgeted run checks its clock between batches. +const BATCH_UNITS: usize = 32; -/// Block text a batch holds before it is scanned: bounds the memory a batch of bulk units (a Hub +/// Block text a batch holds before it is written: bounds the memory a batch of bulk units (a Hub /// shard holds thousands of sessions) takes, where the unit count alone would not. -const SCAN_BATCH_BYTES: usize = 64 << 20; +const BATCH_BYTES: usize = 64 << 20; -/// Read units from the front of `units` until a batch fills — [`SCAN_BATCH`] consumed, or -/// [`SCAN_BATCH_BYTES`] of block text read, whichever comes first. `read(n, unit)` gives the +/// Read units from the front of `units` until a batch fills — [`BATCH_UNITS`] consumed, or +/// [`BATCH_BYTES`] of block text read, whichever comes first. `read(n, unit)` gives the /// `n`th unit's turns, or `None` for one the batch has no use for (already indexed, rejected). /// Either way the unit is consumed and counts, so whatever a backlog holds, a caller's check /// between batches is never more than a batch away. Returns one entry per unit consumed, in @@ -270,25 +202,13 @@ fn take_batch( .sum::(); } batch.push(turns); - if batch.len() >= SCAN_BATCH || bytes >= SCAN_BATCH_BYTES { + if batch.len() >= BATCH_UNITS || bytes >= BATCH_BYTES { break; } } Ok(batch) } -/// The secret scanner, if installed. Best-effort: without one, indexing continues unredacted — the -/// push gate still scans, fail-closed, before any upload, so a secret can't reach the Hub. -fn find_scanner() -> Option { - match scan::Trufflehog::find() { - Ok(s) => Some(s), - Err(e) => { - eprintln!("note: secret redaction disabled — {e}"); - None - } - } -} - /// A unit's distinct-session count and a log label: `" ()"` for a single session (a /// JSONL file), `" sessions"` for a bulk unit (many sessions in one artifact), and the unit's `key` (its /// path) when it has no turns at all. The borrow of `turns` is confined here so callers keep it mutable. @@ -458,8 +378,8 @@ pub(crate) fn local_index_coverage() -> Option { }) } -/// A set-up indexer: it holds the memory lock, embedder, dataset, redaction scanner, and incremental -/// state, so a caller can index units in whatever batches it likes — one at a time to check the +/// A set-up indexer: it holds the memory lock, embedder, dataset, and incremental state, so a +/// caller can index units in whatever batches it likes — one at a time to check the /// clock between them, or all at once — without reloading the model. Build the indexes once at the /// end with [`Indexer::finalize`]. struct Indexer { @@ -468,7 +388,6 @@ struct Indexer { /// [`stored_ids`] at open plus everything written this run — the dedup baseline for new chunks. existing: HashSet, embedder: Box, - scanner: Option, include_thinking: bool, /// cwd → resolved `repo` value, so each distinct checkout runs `git` once across the run. repo_cache: HashMap, @@ -537,7 +456,7 @@ fn collect_units(sources: &[Box]) -> Result>, no_thinking: bool) -> Result { std::fs::create_dir_all(home::dir())?; let interactive = std::io::stdin().is_terminal(); @@ -576,7 +495,6 @@ impl Indexer { }; let embedder: Box = inference::embedder(model)?; - let scanner = find_scanner(); let units = collect_units(&sources)?; if !no_thinking { @@ -598,7 +516,6 @@ impl Indexer { ds, existing, embedder, - scanner, include_thinking: !no_thinking, repo_cache: HashMap::new(), state, @@ -700,16 +617,17 @@ impl Indexer { } } - /// Read and sanitize a batch of units, with one scanner run. Returns one entry per unit - /// consumed: its turns, or `None` when skipped or refused. + /// Read a batch of units, their data URIs elided. Returns one entry per unit consumed: its turns, + /// or `None` when skipped or refused. fn read_batch( &mut self, units: &[usize], label: impl Fn(usize) -> String, ) -> Result>>> { let mut read = take_batch(units, |n, &i| self.read_unit(i, &label(n)))?; - let mut all: Vec<&mut [traces::Turn]> = read.iter_mut().flatten().map(Vec::as_mut_slice).collect(); - sanitize_units(&mut all, self.include_thinking, self.scanner.as_ref())?; + for turns in read.iter_mut().flatten() { + elide_turns(turns); + } Ok(read) } @@ -1212,7 +1130,6 @@ pub fn check(path: &Path, no_thinking: bool, limit: Option) -> Result) -> Result = read.iter_mut().flatten().map(Vec::as_mut_slice).collect(); - sanitize_units(&mut all, !no_thinking, scanner.as_ref())?; + for unit_turns in read.iter_mut().flatten() { + elide_turns(unit_turns); + } let mut rejections = rejections.into_iter(); for (n, unit_turns) in read.iter().enumerate() { let Some(unit_turns) = unit_turns else { @@ -1787,23 +1705,7 @@ mod tests { } #[test] - fn sanitize_elides_payloads_before_the_scanner_sees_them() { - // Records what it is shown, and finds the key wherever it is. - struct Recorder(std::cell::RefCell); - impl scan::SecretScanner for Recorder { - fn scan(&self, texts: &[&str]) -> Result>> { - self.0.borrow_mut().push_str(&texts.join("\n")); - let key = scan::Finding { - detector: "PrivateKey".into(), - raw: "TOPSECRET".into(), - decoder: "PLAIN".into(), - }; - Ok(texts - .iter() - .map(|t| t.contains("TOPSECRET").then(|| key.clone()).into_iter().collect()) - .collect()) - } - } + fn a_stored_block_loses_its_data_uris_and_keeps_its_text_as_written() { let mut turns = vec![traces::Turn { format: traces::FORMAT_VERSION, session_id: "sess".into(), @@ -1823,81 +1725,10 @@ mod tests { source_path: String::new(), harness: "codex".into(), }]; - let scanner = Recorder(std::cell::RefCell::new(String::new())); - sanitize_units(&mut [&mut turns], true, Some(&scanner)).unwrap(); - assert_eq!( - turns[0].blocks[0].text, - r#"{"image_url":"data:image/png;base64,[elided]"} key=[REDACTED:PrivateKey]"# - ); - assert!( - !scanner.0.borrow().contains("iVBORw0KGgo"), - "the scanner must never see the payload: excising a match inside it would strand the rest" - ); - } - - #[test] - fn redact_scans_every_unit_in_one_pass_and_only_the_blocks_stored() { - // Finds the secret a text carries, so a finding must land on the unit it came from. - struct Counting(std::cell::Cell); - impl scan::SecretScanner for Counting { - fn scan(&self, texts: &[&str]) -> Result>> { - self.0.set(self.0.get() + 1); - let finding = |detector: &str, raw: &str| scan::Finding { - detector: detector.into(), - raw: raw.into(), - decoder: "PLAIN".into(), - }; - Ok(texts - .iter() - .map(|t| { - let mut found = Vec::new(); - if t.contains("SECRET") { - found.push(finding("PrivateKey", "SECRET")); - } - if t.contains("TOKEN") { - found.push(finding("Slack", "TOKEN")); - } - found - }) - .collect()) - } - } - let block = |bt: &str, text: &str| traces::Block { - block_type: bt.into(), - text: text.into(), - tool_name: None, - tool_use_id: None, - }; - let turn = |blocks: Vec| traces::Turn { - format: traces::FORMAT_VERSION, - session_id: "sess".into(), - cwd: None, - workdir: "proj".into(), - turn_uuid: "turn".into(), - parent_uuid: None, - seq: 0, - ts: String::new(), - role: "user".into(), - blocks, - source_path: String::new(), - harness: "claude_code".into(), - }; - let mut a = vec![turn(vec![ - block("text", "note SECRET here"), - block("thinking", "SECRET thought"), - ])]; - let mut b = vec![turn(vec![block("tool_result", "output TOKEN dump")])]; - let scanner = Counting(std::cell::Cell::new(0)); - redact_units(&mut [&mut a, &mut b], &scanner, false).unwrap(); - assert_eq!(scanner.0.get(), 1, "one scanner run for both units"); - assert_eq!(a[0].blocks[0].text, "note [REDACTED:PrivateKey] here"); - assert_eq!( - a[0].blocks[1].text, "SECRET thought", - "a thinking block --no-thinking won't store is not scanned" - ); + elide_turns(&mut turns); assert_eq!( - b[0].blocks[0].text, "output [REDACTED:Slack] dump", - "each unit of the batch gets its own secrets, and only those" + turns[0].blocks[0].text, r#"{"image_url":"data:image/png;base64,[elided]"} key=TOPSECRET"#, + "the payload goes, and the push gate is what redacts a secret" ); } @@ -1928,20 +1759,20 @@ mod tests { // Tiny units fill a batch by count; the rest go in the next. let batch = take_batch(&units, |_, _| Ok(Some(turns_of(1)))).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); + assert_eq!(batch.len(), BATCH_UNITS); let rest = take_batch(&units[batch.len()..], |_, _| Ok(Some(turns_of(1)))).unwrap(); - assert_eq!(rest.len(), 40 - SCAN_BATCH); + assert_eq!(rest.len(), 40 - BATCH_UNITS); // A unit the pass has no use for is consumed and counts: a backlog of them still ends a // batch, so a budget check is never more than a batch away. let batch = take_batch(&units, |n, _| Ok((n % 2 == 0).then(|| turns_of(1)))).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); - assert_eq!(batch.iter().flatten().count(), SCAN_BATCH / 2); + assert_eq!(batch.len(), BATCH_UNITS); + assert_eq!(batch.iter().flatten().count(), BATCH_UNITS / 2); let batch = take_batch(&units, |_, _| Ok(None)).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); + assert_eq!(batch.len(), BATCH_UNITS); // Text fills a batch before the count does. - let batch = take_batch(&units, |_, _| Ok(Some(turns_of(SCAN_BATCH_BYTES / 4)))).unwrap(); + let batch = take_batch(&units, |_, _| Ok(Some(turns_of(BATCH_BYTES / 4)))).unwrap(); assert_eq!(batch.len(), 4); } diff --git a/src/main.rs b/src/main.rs index 447e8a68..fd8123f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -211,9 +211,9 @@ enum Cmd { #[arg(value_name = "MEMORY")] memory: Option, }, - /// Redact secrets from your local memory in place — for rows indexed before redaction existed (or - /// flagged by an updated ruleset); needs no source transcript. Cleans the local memory only: it - /// does NOT scrub an already-published remote, which the push gate can only stop adding to. + /// Redact secrets from your local memory in place, such as the blocks a push holds back. Needs no + /// source transcript, and cleans the local memory only: it does NOT scrub an already-published + /// remote, which the push gate can only stop adding to. Scrub, /// Update funes in place: download the latest release binary for this platform and replace the /// running executable, then bring the installed agent integrations to their newest release. diff --git a/tests/scan_batch.rs b/tests/index_batch.rs similarity index 79% rename from tests/scan_batch.rs rename to tests/index_batch.rs index c7feb272..5e40e863 100644 --- a/tests/scan_batch.rs +++ b/tests/index_batch.rs @@ -1,7 +1,6 @@ -//! A directory of many small turns files is scanned for secrets a batch at a time: the scanner is -//! spawned once per batch, not once per file, by `index --check` and by the index itself, and not -//! at all by a rerun that owes nothing. Own test binary: it sets `$FUNES_HOME` and -//! `$FUNES_TRUFFLEHOG`. +//! A directory of many small turns files is read a batch at a time, by `index --check` and by the +//! index itself, and neither runs the secret scanner: push is where secrets are redacted. A rerun +//! that owes nothing adds no rows. Own test binary: it sets `$FUNES_HOME` and `$FUNES_TRUFFLEHOG`. mod support; @@ -19,13 +18,13 @@ fn write_unit(path: &Path, line: &str, now: SystemTime, age: u64) { } #[tokio::test] -async fn a_directory_is_scanned_a_batch_at_a_time() { +async fn a_directory_is_indexed_a_batch_at_a_time_without_a_secret_scan() { let home = tempfile::tempdir().unwrap(); let bin = tempfile::tempdir().unwrap(); let source = tempfile::tempdir().unwrap(); std::env::set_var("FUNES_HOME", home.path()); - // A scanner that finds nothing and counts its spawns in a file beside itself. + // A scanner that counts its spawns in a file beside itself, so a scan by the index would show. let scanner = bin.path().join("trufflehog"); std::fs::write(&scanner, "#!/bin/sh\necho spawn >> \"$0.spawns\"\n").unwrap(); std::fs::set_permissions(&scanner, std::fs::Permissions::from_mode(0o755)).unwrap(); @@ -82,22 +81,21 @@ async fn a_directory_is_scanned_a_batch_at_a_time() { "a rejected unit is reported where it sits:\n{}", report.text ); - assert_eq!( - spawned(), - 2, - "41 units scan in two batches, the rejected one costing none" - ); + assert_eq!(spawned(), 0, "--check runs no secret scan"); - // The index scans the same two batches; a rerun that owes nothing scans nothing. + // The index runs none either, and a rerun that owes nothing adds no rows. std::fs::remove_file(&bad).unwrap(); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - assert_eq!(spawned(), 4); + assert_eq!(spawned(), 0, "the index runs no secret scan"); assert_eq!(support::stored_rows().await, units as usize); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - assert_eq!(spawned(), 4, "an unchanged rerun spawns no scanner"); - assert_eq!(support::stored_rows().await, units as usize, "and adds no rows"); + assert_eq!( + support::stored_rows().await, + units as usize, + "an unchanged rerun adds no rows" + ); } diff --git a/tests/redact_on_index.rs b/tests/redact_on_index.rs deleted file mode 100644 index 50536983..00000000 --- a/tests/redact_on_index.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Gated end-to-end: a secret planted in a transcript is redacted out of the memory at index time, -//! so it never reaches recall — or, via push, the Hub. Skipped unless trufflehog (the scanner) and -//! ssh-keygen (to mint a throwaway key) are both available. - -use std::io::Write; -use std::process::Command; - -#[tokio::test] -async fn planted_key_is_redacted_at_index_time() { - if funes::scan::Trufflehog::find().is_err() { - eprintln!("skip: trufflehog not found"); - return; - } - let home = tempfile::tempdir().unwrap(); - let source = tempfile::tempdir().unwrap(); - std::env::set_var("FUNES_HOME", home.path()); - - // Mint a throwaway private key (never committed) and plant it in a transcript. - let keyfile = home.path().join("throwaway_ed25519"); - let made = Command::new("ssh-keygen") - .args(["-t", "ed25519", "-N", "", "-q", "-f"]) - .arg(&keyfile) - .status() - .map(|s| s.success()) - .unwrap_or(false); - if !made { - eprintln!("skip: ssh-keygen unavailable"); - return; - } - let key = std::fs::read_to_string(&keyfile).unwrap(); - std::fs::remove_file(&keyfile).unwrap(); - // A distinctive slice of the key body — must not survive into the memory. - let key_body = key.lines().nth(1).unwrap().to_string(); - assert!(key_body.len() > 20); - - let session = "redact-session-0001"; - - let content = format!("here is my deploy key, keep it safe:\n{key}"); - let line = serde_json::json!({ - "format": 1, - "session_id": session, - "cwd": "/home/u/dev/demo", - "turn_uuid": "t1", - "seq": 0, - "ts": "2026-01-01T00:00:00Z", - "role": "user", - "blocks": [{"block_type": "text", "text": content}], - "harness": "claude", - }) - .to_string(); - let mut f = std::fs::File::create(source.path().join(format!("{session}.funes.jsonl"))).unwrap(); - writeln!(f, "{line}").unwrap(); - - // Index for real — redaction runs over the block text before chunking/embedding/storing. - funes::commands::index::run_index(source.path(), false, None) - .await - .unwrap(); - - // Read the stored turn back: the marker is present, the key body is gone. - let got = funes::commands::recall::get( - funes::memory::Memory::local(), - session.into(), - funes::commands::recall::TurnRange::default(), - ) - .await - .unwrap(); - assert!( - got.contains("[REDACTED:PrivateKey]"), - "expected a redaction marker, got: {got}" - ); - assert!(!got.contains(&key_body), "key body leaked into the memory: {got}"); -} diff --git a/tests/scrub.rs b/tests/scrub.rs index 0ccf2d82..2aeadcc9 100644 --- a/tests/scrub.rs +++ b/tests/scrub.rs @@ -2,9 +2,8 @@ //! its source, and re-embeds what it rewrote with the memory's own model (multilingual-e5-small //! here). Skipped unless trufflehog and ssh-keygen are available. //! -//! A dirty memory is created by indexing with `FUNES_TRUFFLEHOG=/usr/bin/true` — `true` ignores its -//! args and exits 0, so the scan finds nothing and the secret is stored unredacted (simulating a -//! memory built before redaction existed). Then scrub runs with the real scanner and cleans it. +//! Indexing stores the secret as written, so the memory is dirty until scrub runs with the real +//! scanner and cleans it. mod support; @@ -50,8 +49,6 @@ async fn scrub_redacts_an_existing_secret_in_place_with_the_memorys_model() { let session = "scrub-session-0001"; support::write_session(source.path(), session, &[format!("部署密钥:\n{key}")]); - // Index with a no-op scanner so the secret lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); @@ -68,8 +65,7 @@ async fn scrub_redacts_an_existing_secret_in_place_with_the_memorys_model() { "setup: the key should be on disk before scrub" ); - // Scrub with the real scanner: it must redact the key in place. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Scrub must redact the key in place. funes::commands::scrub::run().await.unwrap(); let clean = recall::get(Memory::local(), session.into(), TurnRange::default()) .await diff --git a/tests/scrub_drops_all.rs b/tests/scrub_drops_all.rs index 397c96d7..05a12f69 100644 --- a/tests/scrub_drops_all.rs +++ b/tests/scrub_drops_all.rs @@ -61,14 +61,12 @@ async fn scrub_dropping_every_block_leaves_a_valid_empty_memory() { let mut f = std::fs::File::create(source.path().join(format!("{session}.funes.jsonl"))).unwrap(); writeln!(f, "{line}").unwrap(); - // Index with a no-op scanner so the escaped key lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); + // Indexing stores the escaped key as written. funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - // Scrub with the real scanner: every block is unredactable, so the memory is rewritten empty. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Every block is unredactable, so scrub rewrites the memory empty. funes::commands::scrub::run().await.unwrap(); // The empty Overwrite must leave a valid, reopenable memory with zero rows. diff --git a/tests/scrub_redacts_escaped_key.rs b/tests/scrub_redacts_escaped_key.rs index 551e55ca..1acaa217 100644 --- a/tests/scrub_redacts_escaped_key.rs +++ b/tests/scrub_redacts_escaped_key.rs @@ -66,8 +66,7 @@ async fn scrub_redacts_an_escaped_key_in_place() { writeln!(f, "{line}").unwrap(); } - // Index with a no-op scanner so the escaped key lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); + // Indexing stores the escaped key as written. funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); @@ -86,8 +85,7 @@ async fn scrub_redacts_an_escaped_key_in_place() { "setup: the escaped key should be in the memory before scrub" ); - // Scrub with the real scanner: it matches the key's JSON-escaped form and excises it in place. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Scrub matches the key's JSON-escaped form and excises it in place. funes::commands::scrub::run().await.unwrap(); let after_key = funes::commands::recall::get( From 73b2858c4d306750bdf0fb9f6b650370f7044da8 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:47:54 +0100 Subject: [PATCH 07/18] review: say in SECURITY.md that push redacts, and indexing stores as written SECURITY.md still said credentials are redacted at index time, and that the gate withholds every row holding a secret. Indexing now stores sessions as written, and the gate redacts what it can and withholds the rest. Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 5cae117e..24658ed4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,15 +27,16 @@ machine. The design keeps you in control of it: ## Secrets -Three layers keep credentials out of a memory and off the Hub: +The local memory keeps sessions as written, credentials included: it is not a publication +boundary. Two layers keep credentials off the Hub: -- **At index time**, credentials are redacted from each session before it is saved. - **At publish time**, an always-on gate scans every outgoing chunk (via - [trufflehog](https://github.com/trufflesecurity/trufflehog)) and *withholds* any row that - still contains a secret, exiting non-zero rather than uploading it. The gate is fail-closed: - if the scanner is missing or crashes, nothing is published. -- **`funes scrub`** removes secrets from rows already in your local memory, so a subsequent - push goes through clean. + [trufflehog](https://github.com/trufflesecurity/trufflehog)) and redacts the secrets it finds + before uploading. A block it can't redact in place is *withheld*, and the push exits non-zero + when that leaves nothing to publish. The gate is fail-closed: if the scanner is missing or + crashes, nothing is published. +- **`funes scrub`** redacts secrets from the rows already in your local memory, including the + blocks a push withholds, so the next push publishes them. ### If a secret reached a remote memory anyway From f7164d6c6c292ebe9b689499a9c82d1699c321a2 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:48:47 +0100 Subject: [PATCH 08/18] review: give every workload document its own text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GPU embedder embeds a repeated text once, and four workloads repeated theirs: 30×~500tok is one text, 100×real and 256×real hold 32, 256×mixed 17. bench_models --backend metal timed those few texts while it reported the whole workload, up to 30x too fast. Each document of those workloads now starts with its number. Co-Authored-By: Claude Opus 5.5 --- benchmark/workloads.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/benchmark/workloads.rs b/benchmark/workloads.rs index 02c9a369..68a95e3a 100644 --- a/benchmark/workloads.rs +++ b/benchmark/workloads.rs @@ -7,6 +7,9 @@ //! measured chunks do, so it shows what a backend spends on padding a mixed batch. The 256-document //! workloads match the production batch size and show how grouping by length scales. The //! 100-document one is recall's pool at `--candidates 100`. +//! +//! No two documents of a workload are the same text: the GPU embedder embeds a repeated text +//! once, so a workload with repeats would time fewer documents than it holds. /// The query every reranker workload scores its documents against. pub const QUERY: &str = "why did we move the reranker off the onnx runtime"; @@ -39,7 +42,7 @@ const SENT: &str = "recall fuses vector ann and bm25 hits by reciprocal rank bef /// `n` documents of about 500 tokens, truncated at 512: recall's rerank candidates at the cap. fn capped_docs(n: usize) -> Vec { - vec![SENT.repeat(18); n] + (0..n).map(|i| format!("{i} {}", SENT.repeat(18))).collect() } /// A ragged batch, two ~512-token docs and the rest short: real indexing's shape, and the one that @@ -69,20 +72,24 @@ pub fn real_docs() -> Vec { fn big_mixed_docs() -> Vec { let short = short_docs(); let mut docs = capped_docs(32); - docs.extend((0..256 - docs.len()).map(|i| short[i % short.len()].clone())); + docs.extend((0..256 - docs.len()).map(|i| format!("{i} {}", short[i % short.len()]))); docs } /// Like `real_docs` but 256 documents, the production batch size, with the same length spread. fn big_real_docs() -> Vec { let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..256).map(|i| words[..REAL_WORDS[i % 32]].join(" ")).collect() + (0..256) + .map(|i| format!("{i} {}", words[..REAL_WORDS[i % 32]].join(" "))) + .collect() } /// Like `real_docs` but 100 documents: recall's pool at `--candidates 100`. fn deep_docs() -> Vec { let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..100).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() + (0..100) + .map(|i| format!("{i} {}", words[..REAL_WORDS[i * 13 % 32]].join(" "))) + .collect() } /// Each workload's label and documents. From 77bf690c6890d66deb3ef7e2dc20d37ceac90ea0 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:49:17 +0100 Subject: [PATCH 09/18] review: describe the workloads, not the programs that run them workloads.rs opened by naming bench_models and score_models, its two users. It also called 256 documents the production batch size, which holds on the CPU only: an indexing call on the GPU takes 256 to 2,048 texts. Co-Authored-By: Claude Opus 5.5 --- benchmark/README.md | 4 ++-- benchmark/workloads.rs | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 6d85dd5d..29d52f5d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -138,8 +138,8 @@ The seven workloads live in `workloads.rs`, which `score_models` shares: - A ragged batch of 2 capped documents among 14 short ones. - 32 and 100 documents whose token lengths are the quantiles of real chunks. 100 is recall's pool at `--candidates 100`. -- 256-document mixed and real batches, the production batch size, which show how a backend scales - and what it spends on padding. +- 256-document mixed and real batches, an indexing call's size on the CPU, which show how a backend + scales and what it spends on padding. Edit the query or the workloads when investigating a particular regression. diff --git a/benchmark/workloads.rs b/benchmark/workloads.rs index 68a95e3a..da5adaec 100644 --- a/benchmark/workloads.rs +++ b/benchmark/workloads.rs @@ -1,12 +1,12 @@ -//! The documents `bench_models` times and `score_models` scores, in seven workloads, because they -//! stress different things. A batch of short docs is dominated by per-call overheads (tokenization, -//! thread spawns), while 30 docs at the 512-token truncation cap, recall's rerank worst case, are -//! dominated by GEMM throughput and memory behavior. The ragged batch is real indexing's shape: -//! batch-longest padding masks most attention columns, whose softmax weights underflow, and the -//! scores×V GEMM then reads what they leave behind. The real batch spreads its lengths the way -//! measured chunks do, so it shows what a backend spends on padding a mixed batch. The 256-document -//! workloads match the production batch size and show how grouping by length scales. The -//! 100-document one is recall's pool at `--candidates 100`. +//! Seven workloads of documents to embed and rerank, because they stress different things. A batch +//! of short docs is dominated by per-call overheads (tokenization, thread spawns), while 30 docs at +//! the 512-token truncation cap, recall's rerank worst case, are dominated by GEMM throughput and +//! memory behavior. The ragged batch is real indexing's shape: batch-longest padding masks most +//! attention columns, whose softmax weights underflow, and the scores×V GEMM then reads what they +//! leave behind. The real batch spreads its lengths the way measured chunks do, so it shows what a +//! backend spends on padding a mixed batch. The 256-document workloads are an indexing call's size +//! on the CPU, and show how grouping by length scales. The 100-document one is recall's pool at +//! `--candidates 100`. //! //! No two documents of a workload are the same text: the GPU embedder embeds a repeated text //! once, so a workload with repeats would time fewer documents than it holds. @@ -68,7 +68,7 @@ pub fn real_docs() -> Vec { (0..32).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() } -/// Like `mixed_docs` but 256 documents, the batch size `embed_batched` uses in production. +/// Like `mixed_docs` but 256 documents, an indexing call's size on the CPU. fn big_mixed_docs() -> Vec { let short = short_docs(); let mut docs = capped_docs(32); @@ -76,7 +76,8 @@ fn big_mixed_docs() -> Vec { docs } -/// Like `real_docs` but 256 documents, the production batch size, with the same length spread. +/// Like `real_docs` but 256 documents, with the same length spread: an indexing call's size on the +/// CPU. fn big_real_docs() -> Vec { let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); (0..256) From 0657faf415816019ce3328d0cb8e228523b96e47 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:49:46 +0100 Subject: [PATCH 10/18] review: claim in the scanner test only what it asserts Its name and doc said a directory is indexed a batch at a time, which no assertion has checked since indexing stopped spawning the scanner once per batch. take_batch's unit test covers how a batch fills. The test now says what it asserts, that indexing runs no secret scanner, and its file is named for it. Co-Authored-By: Claude Opus 5.5 --- tests/{index_batch.rs => index_runs_no_scanner.rs} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename tests/{index_batch.rs => index_runs_no_scanner.rs} (90%) diff --git a/tests/index_batch.rs b/tests/index_runs_no_scanner.rs similarity index 90% rename from tests/index_batch.rs rename to tests/index_runs_no_scanner.rs index 5e40e863..6b4c868d 100644 --- a/tests/index_batch.rs +++ b/tests/index_runs_no_scanner.rs @@ -1,6 +1,6 @@ -//! A directory of many small turns files is read a batch at a time, by `index --check` and by the -//! index itself, and neither runs the secret scanner: push is where secrets are redacted. A rerun -//! that owes nothing adds no rows. Own test binary: it sets `$FUNES_HOME` and `$FUNES_TRUFFLEHOG`. +//! Indexing runs no secret scanner, neither `index --check` nor the index itself, over a directory +//! of more turns files than a batch holds: push is where secrets are redacted. A rerun that owes +//! nothing adds no rows. Own test binary: it sets `$FUNES_HOME` and `$FUNES_TRUFFLEHOG`. mod support; @@ -18,7 +18,7 @@ fn write_unit(path: &Path, line: &str, now: SystemTime, age: u64) { } #[tokio::test] -async fn a_directory_is_indexed_a_batch_at_a_time_without_a_secret_scan() { +async fn indexing_a_directory_runs_no_secret_scan() { let home = tempfile::tempdir().unwrap(); let bin = tempfile::tempdir().unwrap(); let source = tempfile::tempdir().unwrap(); From 2bc8c4280b639405682db55292813fbfaa05a41f Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:50:15 +0100 Subject: [PATCH 11/18] review: leave the secret scan out of what bench_index times bench_index's docs and RowsWritten listed scanning for secrets among the work of writing rows, which indexing no longer does. Co-Authored-By: Claude Opus 5.5 --- benchmark/README.md | 14 ++++++-------- benchmark/bench_index.rs | 11 +++++------ src/commands/index.rs | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 29d52f5d..53f07ac5 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -96,10 +96,9 @@ that floor moves with the CPU and the inference backend. What the benchmark meas `bench_index.rs` times the part of `funes index` that writes rows, in a scratch memory. It leaves the embedding out, since it takes the longest and does not depend on how the rows are written. It -times writing the rows (reading each turns file, scanning it for secrets, chunking it, appending its -rows, recording its state), then the end of the run (the language counts and the compaction). Both -grow with the number of sessions as well as with their size, so a history of many short sessions is -where they show. +times writing the rows (reading each turns file, chunking it, appending its rows, recording its +state), then the end of the run (the language counts and the compaction). Both grow with the number +of sessions as well as with their size, so a history of many short sessions is where they show. ```sh cargo run --release --example bench_index -- --sessions 2000 2>/dev/null @@ -108,10 +107,9 @@ cargo run --release --example bench_index -- --dir path/to/turns --limit 5000 2> The corpus is generated, `--sessions` sessions (2000 by default) of `--turns` turns (12) that are the same on every run, or a directory of turns files given with `--dir`, of which `--limit` keeps -the most recent sessions. The secrets are scanned when TruffleHog is installed, as in -`funes index`. The indexer reports each session and the compaction on stderr. The bench prints the -sessions and chunks written with their time and rate, the fragments the writes left for the -compaction to merge, then the time the end of the run took. +the most recent sessions. The indexer reports each session and the compaction on stderr. The bench +prints the sessions and chunks written with their time and rate, the fragments the writes left for +the compaction to merge, then the time the end of the run took. ## `bench_models` — embedding and reranking time diff --git a/benchmark/bench_index.rs b/benchmark/bench_index.rs index f892c727..0a8095b7 100644 --- a/benchmark/bench_index.rs +++ b/benchmark/bench_index.rs @@ -7,15 +7,14 @@ //! //! An index run writes every session's rows, embeds them, then compacts the memory. The embedding //! takes the longest and does not depend on how the rows are written, so this leaves it out and -//! times the rest: writing the rows (reading each turns file, scanning it for secrets, chunking it, -//! appending its rows, recording its state), then the end of the run (the language counts and the -//! compaction). Both grow with the number of sessions as well as with their size, so a history of -//! many short sessions is where they show. +//! times the rest: writing the rows (reading each turns file, chunking it, appending its rows, +//! recording its state), then the end of the run (the language counts and the compaction). Both +//! grow with the number of sessions as well as with their size, so a history of many short sessions +//! is where they show. //! //! The corpus is generated, `--sessions` sessions of `--turns` turns that are the same on every run, //! or a directory of turns files given with `--dir`, of which `--limit` keeps the most recent -//! sessions. The secrets are scanned when TruffleHog is installed, as in `funes index`. The indexer -//! reports each session and the compaction on stderr. +//! sessions. The indexer reports each session and the compaction on stderr. use std::path::{Path, PathBuf}; diff --git a/src/commands/index.rs b/src/commands/index.rs index da04ed03..825152f6 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -1200,7 +1200,7 @@ pub struct RowsWritten { pub chunks: u64, /// The fragments the writes left for the compaction to merge. pub fragments: usize, - /// Every unit's rows: reading, scanning for secrets, chunking, appending, recording its state. + /// Every unit's rows: reading, chunking, appending, recording its state. pub write: Duration, /// The end of the run: the language counts and the compaction. pub finalize: Duration, From 269820bb5d37778e59853842f83a0e665a15b9c3 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:50:51 +0100 Subject: [PATCH 12/18] review: say why push holds a block back, without an earlier redaction The gate's comments said it redacts the secrets the rows "still" hold, from when indexing redacted first. Skipped said rows are held back for holding a secret, when they are held back only where their block can't be redacted in place. The escaped-key test told the story of the leak it once caught, and the gate's doc kept an em dash. Co-Authored-By: Claude Opus 5.5 --- src/commands/push.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/commands/push.rs b/src/commands/push.rs index be204273..16bbd4da 100644 --- a/src/commands/push.rs +++ b/src/commands/push.rs @@ -2,7 +2,7 @@ //! //! Streamed, never a full mirror. "What's already there" is the memory's own chunk ids, so the //! delta is `embedded_local_ids − remote_ids`. `push` is orchestration: -//! it computes the delta, redacts the secrets still in the rows it ships (holding back the blocks it +//! it computes the delta, redacts the secrets in the rows it ships (holding back the blocks it //! can't redact in place, which wait for `funes scrub`), and drives the HF write operations in //! [`crate::memory::remote`], which own the atomic, parent-commit-guarded commits. //! @@ -402,7 +402,7 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: return Ok(format!("{}: up to date ({} chunks)\n{note}", target.label(), remote_ids.len()).into()); } - // 4. Rows, then redact the secrets they still hold. Re-stamp each batch with the local + // 4. Rows, then redact the secrets they hold. Re-stamp each batch with the local // dataset's schema so its metadata (the embedding-model id) rides along — scan-result batches // drop it, and on first publish that schema is what the new dataset persists. let schema = dataset::schema_of(&local); @@ -416,9 +416,9 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: }; let batches = dataset::restamped(&schema, rows)?; - // Redact the secrets the rows still hold, and hold back the blocks that can't be redacted in - // place rather than block the whole push: `funes scrub` redacts or drops them in the local - // memory, and the next push ships what it leaves. + // Redact the secrets the rows hold, and hold back the blocks that can't be redacted in place + // rather than block the whole push: `funes scrub` redacts or drops them in the local memory, + // and the next push ships what it leaves. let n_scanning: usize = batches.iter().map(|b| b.num_rows()).sum(); eprintln!("scanning {n_scanning} chunk(s) for secrets…"); let gated = gate(&batches)?; @@ -579,7 +579,7 @@ fn card_file(remote_readme: Result>, ctx: &CardCtx) -> (Option Result { let mut gated = Gated { dirty: Vec::new(), @@ -974,10 +974,10 @@ mod tests { #[test] fn an_escaped_key_is_redacted_not_pushed() { - // The exact shape that leaked: a key stored with escaped `\n` (literal backslash-n), as a - // JSON-encoded transcript or a logged blob would hold it. trufflehog still detects it, but - // its canonical `raw` (real newlines) is not a substring of the stored bytes, so value - // matching missed it and pushed it. Its escaped form must be found and redacted. + // A key stored with escaped `\n` (literal backslash-n), as a JSON-encoded transcript or a + // logged blob would hold it. trufflehog detects it, but its canonical `raw` (real newlines) + // is not a substring of the stored bytes, so the gate must find its escaped form to redact + // it. if scan::Trufflehog::find().is_err() { eprintln!("skip: trufflehog not found"); return; From 8494cc2a8d92a764340eff27a1e2ce5d474f37f7 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:51:45 +0100 Subject: [PATCH 13/18] review: keep the index's doc comments to what each thing is Several described their callers instead: write_rows named the bench that times it, record and persist said how often write_units writes the state, and write_unit said what write_units does with its spool file. Why the state is written once a batch now sits where it is written. write_unit no longer calls its turns sanitized, since only their data URIs are elided. BATCH_UNITS gives no reason for 32: the reason was the TruffleHog spawn, which indexing no longer runs. The two lines the branch touched with em dashes lose them. Co-Authored-By: Claude Opus 5.5 --- src/commands/index.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/commands/index.rs b/src/commands/index.rs index 825152f6..a7631a56 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -172,15 +172,14 @@ fn elide_turns(turns: &mut [traces::Turn]) { } } -/// Units a batch consumes. A run reads, writes and records its units a batch at a time, and a -/// budgeted run checks its clock between batches. +/// Units a batch consumes, at most: a batch's units are read, written and recorded together. const BATCH_UNITS: usize = 32; /// Block text a batch holds before it is written: bounds the memory a batch of bulk units (a Hub /// shard holds thousands of sessions) takes, where the unit count alone would not. const BATCH_BYTES: usize = 64 << 20; -/// Read units from the front of `units` until a batch fills — [`BATCH_UNITS`] consumed, or +/// Read units from the front of `units` until a batch fills: [`BATCH_UNITS`] consumed, or /// [`BATCH_BYTES`] of block text read, whichever comes first. `read(n, unit)` gives the /// `n`th unit's turns, or `None` for one the batch has no use for (already indexed, rejected). /// Either way the unit is consumed and counts, so whatever a backlog holds, a caller's check @@ -379,9 +378,9 @@ pub(crate) fn local_index_coverage() -> Option { } /// A set-up indexer: it holds the memory lock, embedder, dataset, and incremental state, so a -/// caller can index units in whatever batches it likes — one at a time to check the -/// clock between them, or all at once — without reloading the model. Build the indexes once at the -/// end with [`Indexer::finalize`]. +/// caller can index units in whatever batches it likes (one at a time to check the clock between +/// them, or all at once) without reloading the model. Build the indexes once at the end with +/// [`Indexer::finalize`]. struct Indexer { uri: String, ds: Option, @@ -561,14 +560,14 @@ impl Indexer { .collect() } - /// Record a unit's state, which [`Indexer::persist`] writes with the rest of its batch. + /// Record a unit's state, unwritten until the next [`Indexer::persist`]. fn record(&mut self, key: &str, state: UnitState) { self.state.insert(key.to_string(), state); self.recorded = true; } - /// Persist the recorded states, so an interrupted run resumes where it stopped. Each write holds - /// every unit's state, so it runs once a batch rather than once a unit, and only after a record. + /// Write every unit's state, if one was recorded since the last write, so an interrupted run + /// resumes where it stopped. fn persist(&mut self) -> Result<()> { if !std::mem::take(&mut self.recorded) { return Ok(()); @@ -631,10 +630,10 @@ impl Indexer { Ok(read) } - /// Write all tiers of unit `i` from sanitized `turns`, keeping only new chunks in one append, - /// and record that its rows are in the memory. Its spool file is drained once - /// [`Indexer::write_units`] has persisted that record, and the memory itself carries any - /// embeddings still owed. A bulk source stays one fragment per unit. + /// Write all tiers of unit `i` from its `turns`, keeping only new chunks in one append, and + /// record that its rows are in the memory. A spool file left with nothing owed is queued to + /// drain. The memory itself carries any embeddings still owed. A bulk source stays one fragment + /// per unit. async fn write_unit(&mut self, i: usize, turns: &[traces::Turn], progress: &str) -> Result { let (key, sig) = { let unit = &self.units[i].1; @@ -707,7 +706,8 @@ impl Indexer { self.write_unit(units[n], &turns, &label(n)).await?; } } - // A spool file is drained only once the state says its unit is indexed. + // One state write per batch, since each write holds every unit's state. A spool file is + // drained only once the state says its unit is indexed. self.persist()?; for (key, sig, progress) in std::mem::take(&mut self.drained) { if let Err(e) = drain(&key, &sig) { @@ -1207,7 +1207,7 @@ pub struct RowsWritten { } /// Index `path` as [`run_index`] does, but leave every embedding pending: the part of a run the -/// embedder has no share in, which `benchmark/bench_index.rs` times on its own. +/// embedder has no share in. pub async fn write_rows(path: &Path, max_sessions: Option) -> Result { let mut indexer = Indexer::open(vec![source::open(path, max_sessions)?], false).await?; let total = indexer.unit_count(); From 3ec1a76b1637fb8c9eadf89c2e1131956b40fc88 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:52:02 +0100 Subject: [PATCH 14/18] review: name the spool files left to drain for what they are The field was called drained, but it holds the spool files of a batch that are not drained yet. Co-Authored-By: Claude Opus 5.5 --- src/commands/index.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/index.rs b/src/commands/index.rs index a7631a56..95e9cf18 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -417,9 +417,9 @@ struct Indexer { rejected: HashSet, /// A unit's state was recorded since the last [`Indexer::persist`]. recorded: bool, - /// The spool files this batch indexed, as key, signature and progress label: drained once + /// The spool files this batch indexed, as key, signature and progress label, to drain once /// [`Indexer::persist`] has recorded them. - drained: Vec<(String, String, String)>, + to_drain: Vec<(String, String, String)>, } /// Enumerate every source's units (each source orders its own — recency-desc), tagged with the @@ -533,7 +533,7 @@ impl Indexer { languages, rejected: HashSet::new(), recorded: false, - drained: Vec::new(), + to_drain: Vec::new(), }) } @@ -687,7 +687,7 @@ impl Indexer { }, ); if drains(&key, Tier::ToolResult, self.include_thinking) { - self.drained.push((key, sig.clone(), progress.to_string())); + self.to_drain.push((key, sig.clone(), progress.to_string())); } } self.n_sessions += sessions; @@ -709,7 +709,7 @@ impl Indexer { // One state write per batch, since each write holds every unit's state. A spool file is // drained only once the state says its unit is indexed. self.persist()?; - for (key, sig, progress) in std::mem::take(&mut self.drained) { + for (key, sig, progress) in std::mem::take(&mut self.to_drain) { if let Err(e) = drain(&key, &sig) { eprintln!("{progress} {key} — indexed, but the spool copy stayed: {e:#}"); } From 2fbb324e3fa837e47b21743013fafd55e3453550 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:52:57 +0100 Subject: [PATCH 15/18] review: call the rows a push holds back held, as the coverage does The gate's result named them held, as the push coverage's field does, and run_push renamed them skipped on the way out. One name now runs through. Co-Authored-By: Claude Opus 5.5 --- src/commands/push.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/commands/push.rs b/src/commands/push.rs index 16bbd4da..1e30345d 100644 --- a/src/commands/push.rs +++ b/src/commands/push.rs @@ -428,21 +428,17 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: embed_batched(embedder.as_mut(), texts, |_| {}) })?; let batches = dataset::restamped(&schema, batches)?; - let Gated { - held: skipped, - redacted, - .. - } = gated; + let Gated { held, redacted, .. } = gated; let n_chunks: usize = batches.iter().map(|b| b.num_rows()).sum(); - if n_chunks == 0 && skipped.rows > 0 { + if n_chunks == 0 && held.rows > 0 { // Everything was held back: nothing reached the Hub. Mark `blocked` so the CLI exits non-zero // — automation must not read this as a successful publish. return Ok(Pushed { report: format!( "{}: nothing published — held back {} row(s) with secrets ({}); run `funes scrub`, then push again\n", target.label(), - skipped.rows, - skipped.summary + held.rows, + held.summary ), blocked: true, }); @@ -501,7 +497,7 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: "{}: {pushed} (commit {oid})\n{card_note}{}{}", target.label(), redacted.note(), - skipped.warning() + held.warning() ) .into()); } @@ -554,7 +550,7 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: out.push_str(&compact_auto(&w).await); } out.push_str(&redacted.note()); - out.push_str(&skipped.warning()); + out.push_str(&held.warning()); Ok(out.into()) } From 953b9b210347a1683a25a9f719490b7085b62b78 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 13:53:20 +0100 Subject: [PATCH 16/18] review: re-wrap the backends paragraph of AGENTS.md Dropping bge-reranker-base from it left a short line mid-paragraph. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 51aa97e2..f804108c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,10 +65,10 @@ default `blas` (src/inference/blas.rs, hand-written forwards on Accelerate/faer) (fastembed/ort). On macOS the default build also carries `metal` (src/inference/metal.rs): the same encoders as MPSGraphs in fp16, which the embedder hands calls of 32+ texts to and the reranker calls of 8+ pairs, falling back to `blas` where no GPU starts. `blas` and `metal` each keep one module per -encoder architecture: `bert.rs` (the embedders) and `modernbert.rs` -(gte-reranker-modernbert-base, the reranker). `onnx` runs the same reranker, from the ONNX export in -gte's Hub repo. CI lints and unit-tests the default build on Linux and on macOS 15, each with the -other's seam compiled out, and `onnx` on Linux: so also run +encoder architecture: `bert.rs` (the embedders) and `modernbert.rs` (gte-reranker-modernbert-base, +the reranker). `onnx` runs the same reranker, from the ONNX export in gte's Hub repo. CI lints and +unit-tests the default build on Linux and on macOS 15, each with the other's seam compiled out, and +`onnx` on Linux: so also run `cargo clippy --all-targets --no-default-features --features onnx` before calling work done. `cargo run --release --features onnx --example score_models` scores `blas` and `metal` against `onnx`, and `cargo run --release --example bench_models -- --backend ` times one of them. From 12e8f4a10b75e0bf670f443eaf7af3274adc544e Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 14:40:21 +0100 Subject: [PATCH 17/18] review: keep a stored block to the pieces it was first stored with Indexing deduplicated chunk by chunk, so a block read again with a longer text kept its stored pieces and took the extra ones. funes 1.12 and older stored a block with its secrets cut out, so this branch re-reading such a session stored the raw tail of the secret behind the redacted head. The push gate rebuilds the block from both, finds no secret without the head, and would publish the tail. A block is now deduplicated by its first piece: one already in the memory, or met earlier in the unit, takes none of the new pieces. An edit re-emitted under its turn_uuid is then deduped away whole, as docs/funes-jsonl.md says. Reproduced with the v1.12.0 release and a 4096-bit RSA key: 33 of its 47 lines were stored after the re-read, and scrub found nothing. With this commit, none are stored. Co-Authored-By: Claude Opus 5.5 --- src/commands/index.rs | 15 ++++++++++++--- tests/turns_file_round_trip.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/commands/index.rs b/src/commands/index.rs index 95e9cf18..a629381d 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -659,12 +659,21 @@ impl Indexer { eprintln!("{progress} {label} — no indexable content"); 0 } else { - // A unit can carry one id twice (a turn re-emitted under its `turn_uuid`); the first wins, - // as it would have had the two arrived in separate runs. + // A block stored once takes no more pieces: a re-read can split it into more than the + // memory holds (an edit re-emitted under its `turn_uuid`, or a block an older funes stored + // with its secrets cut out), and the extra ones would sit behind the old ones. A block's + // first piece stands for it, since its pieces are written together, and the first + // occurrence of a block in a unit wins, as it would have in separate runs. let mut in_batch = HashSet::new(); + let mut new_block = false; let new_chunks: Vec = chunks .into_iter() - .filter(|c| !self.existing.contains(&c.id) && in_batch.insert(c.id.clone())) + .filter(|c| { + if c.split_idx == 0 { + new_block = !self.existing.contains(&c.id) && in_batch.insert(c.id.clone()); + } + new_block + }) .collect(); if new_chunks.is_empty() { eprintln!("{progress} {label} — {total_chunks} chunks, all already indexed"); diff --git a/tests/turns_file_round_trip.rs b/tests/turns_file_round_trip.rs index 90fe6f87..5e6caee4 100644 --- a/tests/turns_file_round_trip.rs +++ b/tests/turns_file_round_trip.rs @@ -193,3 +193,29 @@ async fn a_row_closes_on_the_highest_seq_not_the_latest_stamp() { "the span closes on the highest seq's stamp:\n{listing}" ); } + +/// A block keeps the pieces it was first stored with, whatever text a re-read brings. funes 1.12 +/// and older stored a block with its secrets cut out, so reading its session again can split the +/// block into more pieces than the memory holds, the extra ones carrying the secret's raw tail. +#[tokio::test] +async fn a_stored_block_takes_no_pieces_from_a_longer_reread() { + let _one_at_a_time = HOME.lock().await; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("key.funes.jsonl"); + let mut turns = turns_of("funes_jsonl/valid.funes.jsonl"); + turns.truncate(1); + turns[0].blocks[0].text = "deploy key: [REDACTED:PrivateKey]".into(); + write_turns(&path, &turns); + let (home, first) = index_fresh(dir.path()).await; + + // The block as written, three pieces long, read again because its file changed. + turns[0].blocks[0].text = format!("deploy key: {}", "the raw tail of the secret ".repeat(100)); + write_turns(&path, &turns); + std::env::set_var("FUNES_HOME", home.path()); + run_index(dir.path(), false, None).await.unwrap(); + assert_eq!( + rows().await, + first, + "the stored block takes no piece of the longer text" + ); +} From 632814f02122d7c5de2125b68fbc8f7ce85bbcf6 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Fri, 13 Nov 2026 14:43:11 +0100 Subject: [PATCH 18/18] review: redact a block's tool name along with its text A block's text renders its tool name, which is also a column of its own. The push gate cut a secret found there out of the text, then re-split the text under the original tool name, so the redacted rows uploaded the secret in that column, out of the rescan's sight. Scrub re-splits the same way, so a scrubbed block carried such a secret to the next push too. resplit now takes the tool name, and both callers pass it with the same findings cut out as from the text. Co-Authored-By: Claude Opus 5.5 --- src/chunk.rs | 8 +++++--- src/commands/push.rs | 33 ++++++++++++++++++++++++++++++++- src/commands/scrub.rs | 6 +++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 8fa3c17c..c23fa46b 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -253,8 +253,9 @@ pub(crate) fn reconstruct_blocks(chunks: &[Chunk]) -> Vec<(Vec, String)> /// Re-chunk a block's (already rendered) `text` after it changed — e.g. a secret was redacted out — /// reusing the block's identity (session, turn, block_idx, role, …) from `template`. Ids and /// `split_idx` are regenerated from the new split count, so they stay coordinate-stable. An empty -/// result means nothing indexable survived. -pub(crate) fn resplit(template: &Chunk, text: &str) -> Vec { +/// result means nothing indexable survived. The pieces carry `tool_name`, the name `text` renders, +/// which a redaction of the text may have changed. +pub(crate) fn resplit(template: &Chunk, text: &str, tool_name: Option) -> Vec { split(text) .into_iter() .enumerate() @@ -262,6 +263,7 @@ pub(crate) fn resplit(template: &Chunk, text: &str) -> Vec { id: cid(&template.session_id, &template.turn_uuid, template.block_idx, si as i64), text: piece, split_idx: si as i64, + tool_name: tool_name.clone(), ..(*template).clone() }) .collect() @@ -484,7 +486,7 @@ mod tests { let fresh = chunks_from_turns(std::slice::from_ref(&t), true); let template = fresh[0].clone(); let rendered = render("text", &long, &None); - let re = resplit(&template, &rendered); + let re = resplit(&template, &rendered, None); assert_eq!(re.len(), fresh.len()); for (a, b) in re.iter().zip(&fresh) { assert_eq!(a.id, b.id); diff --git a/src/commands/push.rs b/src/commands/push.rs index 1e30345d..6ea50d3b 100644 --- a/src/commands/push.rs +++ b/src/commands/push.rs @@ -667,7 +667,11 @@ fn gate(batches: &[RecordBatch]) -> Result { gated.dirty[i] = true; } let r = scan::excise(text, findings); - let pieces = chunk::resplit(&chunks[idxs[0]], &r.text); + let tool_name = chunks[idxs[0]] + .tool_name + .as_deref() + .map(|n| scan::excise(n, findings).text); + let pieces = chunk::resplit(&chunks[idxs[0]], &r.text, tool_name); let same_ids = pieces.iter().map(|p| &p.id).eq(idxs.iter().map(|&i| &chunks[i].id)); if r.fully_redacted && same_ids { candidates.push((idxs, findings, r.text, pieces)); @@ -968,6 +972,33 @@ mod tests { assert!(!row.text.contains(body), "no line of the key ships"); } + #[test] + fn a_secret_in_a_tool_name_ships_in_neither_the_text_nor_the_column() { + if scan::Trufflehog::find().is_err() { + eprintln!("skip: trufflehog not found"); + return; + } + let Some(key) = keygen(&["-t", "ed25519"]) else { + eprintln!("skip: ssh-keygen unavailable"); + return; + }; + // The block's text renders its tool name, which is also stored in a column of its own. + let mut call = turn(1, "{}"); + call.blocks[0].block_type = "tool_use".into(); + call.blocks[0].tool_name = Some(key.clone()); + let (b, _) = batch(&[turn(0, "clean note"), call]); + + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!(gated.redacted.blocks, 1, "the block is redacted in place"); + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + let body = key.lines().nth(1).unwrap(); + for row in dataset::chunks(&shipped) { + assert!(!row.text.contains(body), "text: {}", row.text); + let name = row.tool_name.unwrap_or_default(); + assert!(!name.contains(body), "tool_name: {name}"); + } + } + #[test] fn an_escaped_key_is_redacted_not_pushed() { // A key stored with escaped `\n` (literal backslash-n), as a JSON-encoded transcript or a diff --git a/src/commands/scrub.rs b/src/commands/scrub.rs index 0ed216e2..68cf3467 100644 --- a/src/commands/scrub.rs +++ b/src/commands/scrub.rs @@ -67,7 +67,11 @@ pub async fn run() -> Result<()> { // window the next is found in. So a redaction counts only if the result scans clean. let clean = r.fully_redacted && scan::scan_blocks(&[r.text.as_str()], &scanner)?[0].is_empty(); if clean { - replacements.extend(chunk::resplit(&chunks[idxs[0]], &r.text)); + let tool_name = chunks[idxs[0]] + .tool_name + .as_deref() + .map(|n| scan::excise(n, findings).text); + replacements.extend(chunk::resplit(&chunks[idxs[0]], &r.text, tool_name)); redacted_detectors.extend(r.removed_detectors); redacted_blocks += 1; } else {