diff --git a/AGENTS.md b/AGENTS.md index e2a4253d..f804108c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,9 +65,10 @@ default `blas` (src/inference/blas.rs, hand-written forwards on Accelerate/faer) (fastembed/ort). On macOS the default build also carries `metal` (src/inference/metal.rs): the same encoders as MPSGraphs in fp16, which the embedder hands calls of 32+ texts to and the reranker calls of 8+ pairs, falling back to `blas` where no GPU starts. `blas` and `metal` each keep one module per -encoder architecture: `bert.rs` (the embedders, and bge-reranker-base on the CPU) and `modernbert.rs` -(gte-reranker-modernbert-base, the reranker). `onnx` runs the same reranker, from the ONNX export in -gte's Hub repo. CI lints and unit-tests the default build on Linux and on macOS 15, each with the -other's seam compiled out, and `onnx` on Linux: so also run -`cargo clippy --all-targets --no-default-features --features onnx` before calling work done; -`cargo run --release --features onnx --example bench_backends` A/Bs them (latency + agreement). +encoder architecture: `bert.rs` (the embedders) and `modernbert.rs` (gte-reranker-modernbert-base, +the reranker). `onnx` runs the same reranker, from the ONNX export in gte's Hub repo. CI lints and +unit-tests the default build on Linux and on macOS 15, each with the other's seam compiled out, and +`onnx` on Linux: so also run +`cargo clippy --all-targets --no-default-features --features onnx` before calling work done. +`cargo run --release --features onnx --example score_models` scores `blas` and `metal` against +`onnx`, and `cargo run --release --example bench_models -- --backend ` times one of them. diff --git a/Cargo.toml b/Cargo.toml index 5e8ff84f..7f28d63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ metal = [ "dep:half", ] # ONNX inference backend (fastembed/ort). When both backends are compiled in, `blas` is the one -# funes runs; the ONNX types are the reference backend for bench_backends. +# funes runs, and the ONNX types are the reference score_models measures it against. onnx = ["dep:fastembed"] # Benchmarks live under benchmark/ alongside their README; run via `cargo run --example `. @@ -84,11 +84,25 @@ onnx = ["dep:fastembed"] name = "bench_recall" path = "benchmark/bench_recall.rs" -# A/B the inference backends (BLAS vs ONNX as reference). Needs both compiled in: -# cargo run --release --features onnx --example bench_backends +# Time the part of `funes index` that writes rows, without the embedding: +# cargo run --release --example bench_index -- --sessions 2000 2>/dev/null [[example]] -name = "bench_backends" -path = "benchmark/bench_backends.rs" +name = "bench_index" +path = "benchmark/bench_index.rs" + +# Time the embedder and the reranker on one backend, blas or, in a macOS build, metal: +# cargo run --release --example bench_models -- --backend metal +[[example]] +name = "bench_models" +path = "benchmark/bench_models.rs" +required-features = ["blas"] + +# Score the backends against ONNX Runtime, which it needs compiled in: +# cargo run --release --features onnx --example score_models +[[example]] +name = "score_models" +path = "benchmark/score_models.rs" +required-features = ["onnx"] # A default `funes recall` scored against a labeled set of questions. [[example]] diff --git a/README.md b/README.md index 27e640d1..27deb326 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,9 @@ funes recall "..." --memory /funes-memory # read any remote memory f ``` That second form is how you read **someone else's** published memory on a topic, without touching -your own setup. Publishing is guarded: funes redacts credentials at index time, and a separate, -always-on gate refuses to push any chunk that still contains a secret. And because a published memory -is just a dataset, you can try recall **right now**, before indexing anything of your own: +your own setup. Publishing is guarded: an always-on gate redacts credentials from what it publishes, +and holds back what it can't redact. And because a published memory is just a dataset, you can try +recall **right now**, before indexing anything of your own: ```bash funes recall "why is funes append-only" --memory huggingface/funes-memory @@ -185,7 +185,7 @@ back to the CPU where no GPU starts. An ONNX Runtime backend is available as an ```bash cargo build --release --no-default-features --features onnx # ONNX backend instead -cargo run --release --features onnx --example bench_backends # A/B both backends +cargo run --release --features onnx --example score_models # score the built-in backend against it ``` ## Notes diff --git a/SECURITY.md b/SECURITY.md index 5cae117e..24658ed4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,15 +27,16 @@ machine. The design keeps you in control of it: ## Secrets -Three layers keep credentials out of a memory and off the Hub: +The local memory keeps sessions as written, credentials included: it is not a publication +boundary. Two layers keep credentials off the Hub: -- **At index time**, credentials are redacted from each session before it is saved. - **At publish time**, an always-on gate scans every outgoing chunk (via - [trufflehog](https://github.com/trufflesecurity/trufflehog)) and *withholds* any row that - still contains a secret, exiting non-zero rather than uploading it. The gate is fail-closed: - if the scanner is missing or crashes, nothing is published. -- **`funes scrub`** removes secrets from rows already in your local memory, so a subsequent - push goes through clean. + [trufflehog](https://github.com/trufflesecurity/trufflehog)) and redacts the secrets it finds + before uploading. A block it can't redact in place is *withheld*, and the push exits non-zero + when that leaves nothing to publish. The gate is fail-closed: if the scanner is missing or + crashes, nothing is published. +- **`funes scrub`** redacts secrets from the rows already in your local memory, including the + blocks a push withholds, so the next push publishes them. ### If a secret reached a remote memory anyway diff --git a/benchmark/README.md b/benchmark/README.md index 22f1d0a5..53f07ac5 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,9 +1,12 @@ # Benchmarks -Four runnable examples, each `cargo run --release --example `: +Six runnable examples, each `cargo run --release --example `: - **`bench_recall`** — `recall()` latency, local vs remote, cold vs warm. -- **`bench_backends`** — latency and output agreement between the inference backends. +- **`bench_index`** — the part of `funes index` that writes rows, without the embedding. +- **`bench_models`** — embedding and reranking time on one inference backend. +- **`score_models`** — how closely the inference backends reproduce ONNX Runtime's embeddings and + reranker scores. - **`score_recall`** — `funes recall` scored against a labeled set of questions. - **`sweep_rerank`** — how much reranking helps at each pool size, on a `dacorvo/funes-recall-benchmark` set. @@ -89,34 +92,69 @@ that floor moves with the CPU and the inference backend. What the benchmark meas - For a private dataset, a token must be available (`HF_TOKEN` or the cached login) — the same one recall uses. -## `bench_backends` — inference backend comparison +## `bench_index` — writing an index's rows -`bench_backends.rs` compares every compiled `Embedder`/`Reranker` implementation. Build with both -the default BLAS backend and the optional ONNX reference: +`bench_index.rs` times the part of `funes index` that writes rows, in a scratch memory. It leaves +the embedding out, since it takes the longest and does not depend on how the rows are written. It +times writing the rows (reading each turns file, chunking it, appending its rows, recording its +state), then the end of the run (the language counts and the compaction). Both grow with the number +of sessions as well as with their size, so a history of many short sessions is where they show. ```sh -cargo run --release --features onnx --example bench_backends -cargo run --release --features onnx --example bench_backends -- intfloat/multilingual-e5-small +cargo run --release --example bench_index -- --sessions 2000 2>/dev/null +cargo run --release --example bench_index -- --dir path/to/turns --limit 5000 2>/dev/null ``` -It runs seven fixed workloads: 16 short documents to expose per-call overhead; 30 documents near -the 512-token truncation limit to approximate recall's rerank worst case; a ragged batch of 2 capped -documents among 14 short ones; 32 and 100 documents whose token lengths are the quantiles of real -chunks (100 is recall's pool at `--candidates 100`); and 256-document mixed and real batches, the -production batch size, which show how a backend scales and what it spends on padding. +The corpus is generated, `--sessions` sessions (2000 by default) of `--turns` turns (12) that are +the same on every run, or a directory of turns files given with `--dir`, of which `--limit` keeps +the most recent sessions. The indexer reports each session and the compaction on stderr. The bench +prints the sessions and chunks written with their time and rate, the fragments the writes left for +the compaction to merge, then the time the end of the run took. -Rerankers are compared within a model: gte-reranker-modernbert-base on ONNX, BLAS and, on macOS, -Metal, and bge-reranker-base on BLAS as the latency baseline. Each reranker's startup comes first: how -long it took to load, then its first call on the 32 real documents, which a one-shot `funes recall` -pays in full. Then each workload's latency, and agreement with the first backend of the same model: +## `bench_models` — embedding and reranking time -- `cos↔ref` is the minimum cosine similarity between corresponding embedding vectors. -- `Δ↔ref` is the maximum absolute difference between corresponding reranker scores. +`bench_models.rs` times funes's embedder and reranker (gte-reranker-modernbert-base) on one +backend: `--backend blas`, the default, runs the CPU forwards every build carries, and +`--backend metal` the GPU forwards of a macOS build. `--model` names the embedding model, +`BAAI/bge-small-en-v1.5` when omitted. -ONNX is the reference when that feature is present, BLAS otherwise. The one argument names the -embedding model, `BAAI/bge-small-en-v1.5` when omitted. Edit the fixed query or workloads when investigating a -particular regression. Building with only one backend still prints its timing, along with a note -that there is nothing to compare. +```sh +cargo run --release --example bench_models -- --backend metal +cargo run --release --example bench_models -- --model intfloat/multilingual-e5-small +``` + +It times the forwards themselves. funes runs them behind a router that keeps a call on the CPU +until it holds 32 texts or 8 pairs, and the two backends' startup and first calls are what those +thresholds weigh. Each model's startup comes first: how long it took to load, then its first call +on the 32 real documents, which a one-shot process such as `funes recall` pays in full. Then each +workload's time per call and documents per second, after one warm-up call. + +The seven workloads live in `workloads.rs`, which `score_models` shares: + +- 16 short documents, to expose per-call overhead. +- 30 documents near the 512-token truncation limit, recall's rerank worst case. +- A ragged batch of 2 capped documents among 14 short ones. +- 32 and 100 documents whose token lengths are the quantiles of real chunks. 100 is recall's pool + at `--candidates 100`. +- 256-document mixed and real batches, an indexing call's size on the CPU, which show how a backend + scales and what it spends on padding. + +Edit the query or the workloads when investigating a particular regression. + +## `score_models` — the backends against ONNX + +`score_models.rs` measures how closely funes's backends reproduce ONNX Runtime running each model's +ONNX export: the CPU forwards (`blas`) and, in a macOS build, the GPU forwards (`metal`), over the +same seven workloads. ONNX is the reference, so the example needs it compiled in: + +```sh +cargo run --release --features onnx --example score_models +cargo run --release --features onnx --example score_models -- --model intfloat/multilingual-e5-small +``` + +- `cos↔onnx` is the lowest cosine similarity between a document's embedding on the backend and on + ONNX. +- `Δ↔onnx` is the largest absolute difference between their reranker scores. ## `score_recall` — recall against labeled questions diff --git a/benchmark/bench_backends.rs b/benchmark/bench_backends.rs deleted file mode 100644 index 69445f84..00000000 --- a/benchmark/bench_backends.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! A/B the inference backends behind the `Embedder`/`Reranker` traits: latency + agreement (does a -//! faster backend embed/rank the same as the reference?). Backend-agnostic and cross-platform — it -//! compares whatever backends are compiled in, with ONNX (fastembed) as the reference when present: -//! cargo run --release --features onnx --example bench_backends [-- ] -//! -//! Rerankers are compared within a model: gte-reranker-modernbert-base on ONNX (the reference), -//! BLAS and, on macOS, Metal, and bge-reranker-base on BLAS as the latency baseline. Each -//! reranker's startup comes first: loading it, then its first call, which a one-shot `funes recall` -//! pays in full (the GPU compiles a graph per batch shape on first use). -//! -//! Seven workloads, because they stress different things: a batch of short docs is dominated by -//! per-call overheads (tokenization, thread spawns), while 30 docs at the 512-token truncation -//! cap — recall's rerank worst case — is dominated by GEMM throughput and memory behavior. The -//! ragged batch is real indexing's shape — batch-longest padding masks most attention columns, -//! whose softmax weights underflow, and the scores×V GEMM then reads what they leave behind. The -//! real batch spreads its lengths the way measured chunks do, so it shows what a backend spends -//! on padding a mixed batch. The 256-document workloads match the production batch size and show -//! how grouping by length scales. The 100-document one is recall's pool at `--candidates 100`. -//! -//! Adding a backend = impl Embedder or Reranker, gate it behind a feature, and push it in -//! `embedders()` or `rerankers()`. - -use std::time::Instant; - -use anyhow::{anyhow, Result}; -use funes::inference::{Embedder, EmbeddingModel, Reranker}; - -const QUERY: &str = "why did we move the reranker off the onnx runtime"; - -fn short_docs() -> Vec { - [ - "the reranker is a cross-encoder scoring query-passage pairs jointly.", - "onnx runtime uses MLAS on the CPU, which on Apple Silicon runs on NEON, not AMX.", - "a hand-written forward calls Accelerate cblas_sgemm, which reaches the AMX matrix units.", - "the embedding model is bge-small-en-v1.5, a 384-dimensional BERT sentence encoder.", - "recall fuses vector ANN and BM25 hits by reciprocal rank before reranking.", - "the memory is a lance dataset with an IVF_PQ vector index and a full-text index.", - "candle's metal backend was missing a layer-norm kernel, so it could not run the model.", - "the cat knocked a glass off the counter and it shattered on the floor.", - "quarterly revenue rose twelve percent on strong subscription renewals.", - "tokenization uses the huggingface tokenizers crate loading the model's tokenizer.json.", - "softmax over the attention scores uses a vectorized exp from the platform seam.", - "the recipe needs two cups of flour, a teaspoon of salt, and three eggs.", - "fp8 has no hardware path on apple silicon, so int8 is the only accelerated low-precision.", - "attention masks let the transformer ignore padding tokens in a ragged batch.", - "the marathon route winds through six neighborhoods before the riverside finish.", - "hf-hub fetches whole files because the xet cdn taxes every byte-range read.", - ] - .map(String::from) - .to_vec() -} - -const SENT: &str = "recall fuses vector ann and bm25 hits by reciprocal rank before the cross-encoder \ - rescores each candidate against the query using joint attention over the pair. "; - -fn capped_docs(n: usize) -> Vec { - // ~500 tokens after tokenization, truncated at 512 — recall's rerank candidates at the cap. - let doc = SENT.repeat(18); - vec![doc; n] -} - -fn long_docs() -> Vec { - capped_docs(30) -} - -/// A ragged batch — a couple of ~512-token docs, the rest short: real indexing's shape, and the -/// one that exposes padding stalls. -fn mixed_docs() -> Vec { - let short = short_docs(); - let mut docs = capped_docs(2); - docs.extend((0..16 - docs.len()).map(|i| short[i % short.len()].clone())); - docs -} - -/// Words of `SENT` whose embedder token counts land on the 32 quantiles of real chunk lengths, -/// measured over 12,971 chunks of 100 Claude Code sessions: 21 up to the 512 cap, median 314. -const REAL_WORDS: [usize; 32] = [ - 13, 30, 38, 51, 66, 87, 106, 126, 145, 163, 176, 187, 196, 205, 213, 218, 226, 232, 238, 247, 255, 262, 269, 280, - 288, 296, 306, 314, 327, 340, 362, 363, -]; - -/// One doc per quantile of real chunk lengths, in a scrambled order — the length spread indexing -/// hands `embed`, and about recall's 30 rerank candidates. -fn real_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..32).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() -} - -/// Like `mixed_docs` but 256 documents — the batch size `embed_batched` uses in production. -fn big_mixed_docs() -> Vec { - let short = short_docs(); - let mut docs = capped_docs(32); - docs.extend((0..256 - docs.len()).map(|i| short[i % short.len()].clone())); - docs -} - -/// Like `real_docs` but 256 documents — the production batch size, with the same length spread. -fn big_real_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..256).map(|i| words[..REAL_WORDS[i % 32]].join(" ")).collect() -} - -/// Like `real_docs` but 100 documents: recall's pool at `--candidates 100`. -fn deep_docs() -> Vec { - let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); - (0..100).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() -} - -/// The embedders compiled in, ONNX first when it is: the first is the agreement reference. -fn embedders(model: EmbeddingModel) -> Result)>> { - let mut v: Vec<(&'static str, Box)> = Vec::new(); - #[cfg(feature = "onnx")] - { - v.push(("onnx", Box::new(funes::inference::OnnxEmbedder::new(model)?))); - } - #[cfg(feature = "blas")] - { - v.push(("blas", Box::new(funes::inference::blas::BlasEmbedder::new(model)?))); - } - Ok(v) -} - -/// A reranker under test, and how long it took to load. -struct Ranker { - model: &'static str, - backend: &'static str, - rr: Box, - load_ms: f64, -} - -/// The rerankers compiled in, by model: the first of a model is the reference for the others. -fn rerankers() -> Result> { - fn load( - model: &'static str, - backend: &'static str, - make: impl FnOnce() -> Result>, - ) -> Result { - let t = Instant::now(); - let rr = make()?; - let load_ms = t.elapsed().as_secs_f64() * 1000.0; - Ok(Ranker { - model, - backend, - rr, - load_ms, - }) - } - let mut v = Vec::new(); - #[cfg(feature = "onnx")] - v.push(load("gte", "onnx", || { - Ok(Box::new(funes::inference::OnnxReranker::new()?)) - })?); - #[cfg(feature = "blas")] - v.push(load("gte", "blas", || { - Ok(Box::new(funes::inference::blas::GteReranker::new()?)) - })?); - #[cfg(all(feature = "metal", target_os = "macos"))] - v.push(load("gte", "metal", || { - Ok(Box::new(funes::inference::metal::MetalReranker::new()?)) - })?); - #[cfg(feature = "blas")] - v.push(load("bge", "blas", || { - Ok(Box::new(funes::inference::blas::BlasReranker::new()?)) - })?); - Ok(v) -} - -fn time(warmups: usize, iters: usize, mut f: F) -> f64 { - for _ in 0..warmups { - f(); - } - let t = Instant::now(); - for _ in 0..iters { - f(); - } - t.elapsed().as_secs_f64() * 1000.0 / iters as f64 -} - -fn cosine(a: &[f32], b: &[f32]) -> f32 { - let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); - let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); - let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); - dot / (na * nb) -} - -fn main() -> Result<()> { - let model = match std::env::args().nth(1) { - Some(id) => EmbeddingModel::from_id(&id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, - None => EmbeddingModel::BgeSmallEn, - }; - let mut embs = embedders(model)?; - let mut rrs = rerankers()?; - if embs.len() < 2 { - eprintln!("note: only one backend is compiled — build with `--features onnx` to A/B against the reference\n"); - } - - // (label, docs, warmups, timed iters) — few iterations where a single forward runs seconds, - // except the ragged batch: it carries the padding behavior worth measuring, and one timed - // iteration cannot resolve a difference of a few percent. - let workloads = [ - ("16×short", short_docs(), 2, 5), - ("30×~500tok", long_docs(), 1, 3), - ("16×mixed", mixed_docs(), 1, 5), - ("32×real", real_docs(), 1, 3), - ("100×real", deep_docs(), 1, 3), - ("256×mixed", big_mixed_docs(), 1, 1), - ("256×real", big_real_docs(), 1, 1), - ]; - - let real = real_docs(); - let real: Vec<&str> = real.iter().map(String::as_str).collect(); - println!("reranker startup: load, then a first call on 32×real"); - println!( - "{:<6} {:<8} {:>10} {:>14}", - "model", "backend", "load ms", "first call ms" - ); - for r in rrs.iter_mut() { - let t = Instant::now(); - r.rr.rerank(QUERY, &real)?; - let first_ms = t.elapsed().as_secs_f64() * 1000.0; - println!("{:<6} {:<8} {:>10.0} {:>14.0}", r.model, r.backend, r.load_ms, first_ms); - } - - println!("\nembedding: {}", model.id()); - println!( - "{:<12} {:<8} {:>10} {:>16}", - "workload", "backend", "embed ms", "cos↔ref" - ); - for (label, docs, warmups, iters) in &workloads { - let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); - // Run each backend once for correctness, then time it. Reference = the first backend. - let mut reference: Option>> = None; - for (name, emb) in embs.iter_mut() { - let e = emb.embed(&docs)?; - let ms = time(*warmups, *iters, || { - emb.embed(&docs).unwrap(); - }); - let cos = match &reference { - None => "(ref)".to_string(), - Some(r) => { - let min = e.iter().zip(r).map(|(a, b)| cosine(a, b)).fold(1f32, f32::min); - format!("{min:.6}") - } - }; - println!("{label:<12} {name:<8} {ms:>10.1} {cos:>16}"); - reference.get_or_insert(e); - } - } - - println!("\nreranking"); - println!( - "{:<12} {:<6} {:<8} {:>10} {:>16}", - "workload", "model", "backend", "rerank ms", "Δ↔ref" - ); - for (label, docs, warmups, iters) in &workloads { - let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); - // Run each reranker once for correctness, then time it. Reference = the model's first. - let mut references: Vec<(&str, Vec)> = Vec::new(); - for r in rrs.iter_mut() { - let scores = r.rr.rerank(QUERY, &docs)?; - let ms = time(*warmups, *iters, || { - r.rr.rerank(QUERY, &docs).unwrap(); - }); - let diff = match references.iter().find(|(m, _)| *m == r.model) { - None => "(ref)".to_string(), - Some((_, want)) => { - let d = scores.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); - format!("{d:.6}") - } - }; - println!("{label:<12} {:<6} {:<8} {ms:>10.1} {diff:>16}", r.model, r.backend); - if !references.iter().any(|(m, _)| *m == r.model) { - references.push((r.model, scores)); - } - } - } - Ok(()) -} diff --git a/benchmark/bench_index.rs b/benchmark/bench_index.rs new file mode 100644 index 00000000..0a8095b7 --- /dev/null +++ b/benchmark/bench_index.rs @@ -0,0 +1,164 @@ +//! Time the part of `funes index` that writes rows, in a scratch memory: +//! +//! ```text +//! cargo run --release --example bench_index -- --sessions 2000 2>/dev/null +//! cargo run --release --example bench_index -- --dir path/to/turns --limit 5000 2>/dev/null +//! ``` +//! +//! An index run writes every session's rows, embeds them, then compacts the memory. The embedding +//! takes the longest and does not depend on how the rows are written, so this leaves it out and +//! times the rest: writing the rows (reading each turns file, chunking it, appending its rows, +//! recording its state), then the end of the run (the language counts and the compaction). Both +//! grow with the number of sessions as well as with their size, so a history of many short sessions +//! is where they show. +//! +//! The corpus is generated, `--sessions` sessions of `--turns` turns that are the same on every run, +//! or a directory of turns files given with `--dir`, of which `--limit` keeps the most recent +//! sessions. The indexer reports each session and the compaction on stderr. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use clap::Parser; + +#[derive(Parser)] +struct Args { + /// Sessions to generate. + #[arg(long, default_value_t = 2000, conflicts_with = "dir")] + sessions: usize, + /// Turns per generated session. + #[arg(long, default_value_t = 12, conflicts_with = "dir")] + turns: usize, + /// A directory of turns files to index instead of a generated corpus. + #[arg(long)] + dir: Option, + /// Index only the directory's most recent N sessions. + #[arg(long, requires = "dir")] + limit: Option, +} + +/// The words a generated turn is drawn from. +const WORDS: [&str; 48] = [ + "the", + "index", + "writes", + "rows", + "for", + "each", + "session", + "and", + "then", + "embeds", + "them", + "while", + "recall", + "reads", + "chunks", + "back", + "from", + "a", + "memory", + "on", + "disk", + "with", + "its", + "turns", + "in", + "order", + "so", + "an", + "agent", + "can", + "find", + "what", + "was", + "decided", + "before", + "about", + "this", + "repository", + "build", + "test", + "run", + "error", + "fix", + "plan", + "review", + "answer", + "question", + "user", +]; + +/// An xorshift generator with a fixed seed, so a generated corpus is the same on every run. +struct Rng(u64); + +impl Rng { + fn below(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % n as u64) as usize + } +} + +/// Write `sessions` turns files of `turns` turns each into `dir`, a turn of 40 to 400 words. +fn generate(dir: &Path, sessions: usize, turns: usize) -> Result<()> { + std::fs::create_dir_all(dir)?; + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for s in 0..sessions { + let session_id = format!("bench-{s:06}"); + let mut file = String::new(); + for seq in 0..turns { + let words: Vec<&str> = (0..40 + rng.below(361)) + .map(|_| WORDS[rng.below(WORDS.len())]) + .collect(); + let turn = serde_json::json!({ + "session_id": session_id, + "turn_uuid": format!("t{seq}"), + "seq": seq, + "ts": "2026-01-01T00:00:00Z", + "role": if seq % 2 == 0 { "user" } else { "assistant" }, + "harness": "bench", + "blocks": [{"block_type": "text", "text": words.join(" ")}], + }); + file.push_str(&turn.to_string()); + file.push('\n'); + } + std::fs::write(dir.join(format!("{session_id}.funes.jsonl")), file)?; + } + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + let scratch = tempfile::tempdir()?; + let (dir, corpus) = match &args.dir { + Some(dir) => { + let kept = args.limit.map(|n| format!(", its {n} most recent sessions")); + (dir.clone(), format!("{}{}", dir.display(), kept.unwrap_or_default())) + } + None => { + let dir = scratch.path().join("turns"); + generate(&dir, args.sessions, args.turns)?; + ( + dir, + format!("{} generated sessions of {} turns", args.sessions, args.turns), + ) + } + }; + // A scratch home: the run builds a memory of its own and leaves yours alone. + std::env::set_var("FUNES_HOME", scratch.path().join("funes")); + let run = funes::commands::index::write_rows(&dir, args.limit).await?; + let write = run.write.as_secs_f64(); + println!("\ncorpus: {corpus}"); + println!( + "rows: {} sessions, {} chunks in {write:.1} s, {:.1} sessions/s", + run.sessions, + run.chunks, + run.sessions as f64 / write + ); + println!("left: {} fragments to merge", run.fragments); + println!("end of run: {:.1} s", run.finalize.as_secs_f64()); + Ok(()) +} diff --git a/benchmark/bench_models.rs b/benchmark/bench_models.rs new file mode 100644 index 00000000..8c1ebe33 --- /dev/null +++ b/benchmark/bench_models.rs @@ -0,0 +1,134 @@ +//! Time funes's embedder and reranker on one inference backend: +//! +//! ```text +//! cargo run --release --example bench_models -- --backend metal +//! ``` +//! +//! `--backend blas` (the default) times the CPU forwards every build carries, `--backend metal` the +//! GPU forwards of a macOS build. These are the forwards themselves: funes runs them behind a router +//! that keeps a call on the CPU until it holds `GPU_MIN_TEXTS` texts or `GPU_MIN_PAIRS` pairs, and +//! the two backends' startup and first calls are what those thresholds weigh. Each model's startup +//! comes first: loading it, then its first call, which a one-shot process pays in full (the GPU +//! uploads its weights and compiles a graph per batch shape on first use). Then each workload's time +//! per call, after one warm-up call. + +#[path = "workloads.rs"] +mod workloads; + +use std::time::Instant; + +use anyhow::{anyhow, Result}; +use clap::{Parser, ValueEnum}; +use funes::inference::{Embedder, EmbeddingModel, Reranker}; + +#[derive(Parser)] +struct Args { + /// The backend whose forwards are timed. + #[arg(long, value_enum, default_value_t = Backend::Blas)] + backend: Backend, + /// The embedding model, by its Hub id. `BAAI/bge-small-en-v1.5` when omitted. + #[arg(long)] + model: Option, +} + +#[derive(Clone, Copy, ValueEnum)] +enum Backend { + /// The CPU forwards, in every build. + Blas, + /// The GPU forwards, in a macOS build. + #[cfg(all(feature = "metal", target_os = "macos"))] + Metal, +} + +impl Backend { + fn name(self) -> &'static str { + match self { + Backend::Blas => "blas", + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => "metal", + } + } + + fn embedder(self, model: EmbeddingModel) -> Result> { + Ok(match self { + Backend::Blas => Box::new(funes::inference::blas::BlasEmbedder::new(model)?), + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => Box::new(funes::inference::metal::MetalEmbedder::new(model)?), + }) + } + + fn reranker(self) -> Result> { + Ok(match self { + Backend::Blas => Box::new(funes::inference::blas::GteReranker::new()?), + #[cfg(all(feature = "metal", target_os = "macos"))] + Backend::Metal => Box::new(funes::inference::metal::MetalReranker::new()?), + }) + } +} + +/// Timed calls for a workload of `n` documents: fewer where a call runs seconds. Up to 16 documents +/// get 5, since the ragged batch carries the padding behavior worth measuring and one call cannot +/// resolve a difference of a few percent. +fn iterations(n: usize) -> usize { + match n { + 0..=16 => 5, + 17..=100 => 3, + _ => 1, + } +} + +/// What `f` returns, and the milliseconds it took. +fn timed(f: impl FnOnce() -> Result) -> Result<(T, f64)> { + let t = Instant::now(); + let out = f()?; + Ok((out, t.elapsed().as_secs_f64() * 1000.0)) +} + +/// Milliseconds per call of `f` over `iters` calls, after one warm-up call. +fn per_call(iters: usize, mut f: impl FnMut() -> Result<()>) -> Result { + f()?; + let ((), ms) = timed(|| (0..iters).try_for_each(|_| f()))?; + Ok(ms / iters as f64) +} + +fn main() -> Result<()> { + let args = Args::parse(); + let model = match &args.model { + Some(id) => EmbeddingModel::from_id(id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, + None => EmbeddingModel::BgeSmallEn, + }; + let real = workloads::real_docs(); + let real: Vec<&str> = real.iter().map(String::as_str).collect(); + + let (mut emb, emb_load) = timed(|| args.backend.embedder(model))?; + let (_, emb_first) = timed(|| emb.embed(&real))?; + let (mut rr, rr_load) = timed(|| args.backend.reranker())?; + let (_, rr_first) = timed(|| rr.rerank(workloads::QUERY, &real))?; + println!( + "{} backend: {} and gte-reranker-modernbert-base", + args.backend.name(), + model.id() + ); + println!("\nstartup: load, then a first call on 32×real"); + println!("{:<10} {:>10} {:>14}", "model", "load ms", "first call ms"); + println!("{:<10} {emb_load:>10.0} {emb_first:>14.0}", "embedder"); + println!("{:<10} {rr_load:>10.0} {rr_first:>14.0}", "reranker"); + + println!( + "\n{:<12} {:>10} {:>10} {:>11} {:>10}", + "workload", "embed ms", "texts/s", "rerank ms", "pairs/s" + ); + for (label, docs) in workloads::workloads() { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let iters = iterations(docs.len()); + let embed_ms = per_call(iters, || emb.embed(&docs).map(drop))?; + let rerank_ms = per_call(iters, || rr.rerank(workloads::QUERY, &docs).map(drop))?; + let per_s = |ms: f64| docs.len() as f64 * 1000.0 / ms; + println!( + "{label:<12} {embed_ms:>10.1} {:>10.0} {rerank_ms:>11.1} {:>10.0}", + per_s(embed_ms), + per_s(rerank_ms) + ); + } + Ok(()) +} diff --git a/benchmark/score_models.rs b/benchmark/score_models.rs new file mode 100644 index 00000000..f7ea3a08 --- /dev/null +++ b/benchmark/score_models.rs @@ -0,0 +1,100 @@ +//! Score funes's inference backends against ONNX Runtime running each model's ONNX export: how +//! closely the CPU forwards (`blas`) and, in a macOS build, the GPU forwards (`metal`) reproduce its +//! embeddings and its reranker scores, over the workloads `bench_models` times. ONNX is the +//! reference, so the example needs it compiled in: +//! +//! ```text +//! cargo run --release --features onnx --example score_models +//! ``` +//! +//! `cos↔onnx` is the lowest cosine similarity between a document's embedding on the backend and on +//! ONNX, and `Δ↔onnx` the largest absolute difference between their reranker scores. + +#[path = "workloads.rs"] +mod workloads; + +use anyhow::{anyhow, Result}; +use clap::Parser; +use funes::inference::{Embedder, EmbeddingModel, OnnxEmbedder, OnnxReranker, Reranker}; + +#[derive(Parser)] +struct Args { + /// The embedding model, by its Hub id. `BAAI/bge-small-en-v1.5` when omitted. + #[arg(long)] + model: Option, +} + +/// A backend scored against ONNX: its name, its embedder and its reranker. +type Backend = (&'static str, Box, Box); + +/// The backends compiled in besides ONNX. +#[cfg(feature = "blas")] +fn backends(model: EmbeddingModel) -> Result> { + Ok(vec![ + ( + "blas", + Box::new(funes::inference::blas::BlasEmbedder::new(model)?) as Box, + Box::new(funes::inference::blas::GteReranker::new()?) as Box, + ), + #[cfg(all(feature = "metal", target_os = "macos"))] + ( + "metal", + Box::new(funes::inference::metal::MetalEmbedder::new(model)?) as Box, + Box::new(funes::inference::metal::MetalReranker::new()?) as Box, + ), + ]) +} + +/// A build with ONNX alone has no other backend to score. +#[cfg(not(feature = "blas"))] +fn backends(_: EmbeddingModel) -> Result> { + Ok(Vec::new()) +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum(); + let na: f32 = a.iter().map(|x| x * x).sum::().sqrt(); + let nb: f32 = b.iter().map(|x| x * x).sum::().sqrt(); + dot / (na * nb) +} + +fn main() -> Result<()> { + let args = Args::parse(); + let model = match &args.model { + Some(id) => EmbeddingModel::from_id(id).ok_or_else(|| anyhow!("funes does not run {id:?}"))?, + None => EmbeddingModel::BgeSmallEn, + }; + let mut backends = backends(model)?; + if backends.is_empty() { + println!("nothing to score: this build has no backend besides ONNX"); + return Ok(()); + } + let mut onnx_embedder = OnnxEmbedder::new(model)?; + let mut onnx_reranker = OnnxReranker::new()?; + let workloads = workloads::workloads(); + + println!("embedding: {}", model.id()); + println!("{:<12} {:<8} {:>12}", "workload", "backend", "cos↔onnx"); + for (label, docs) in &workloads { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let want = onnx_embedder.embed(&docs)?; + for (name, embedder, _) in backends.iter_mut() { + let got = embedder.embed(&docs)?; + let min = got.iter().zip(&want).map(|(a, b)| cosine(a, b)).fold(1f32, f32::min); + println!("{label:<12} {name:<8} {min:>12.6}"); + } + } + + println!("\nreranking: gte-reranker-modernbert-base"); + println!("{:<12} {:<8} {:>12}", "workload", "backend", "Δ↔onnx"); + for (label, docs) in &workloads { + let docs: Vec<&str> = docs.iter().map(String::as_str).collect(); + let want = onnx_reranker.rerank(workloads::QUERY, &docs)?; + for (name, _, reranker) in backends.iter_mut() { + let got = reranker.rerank(workloads::QUERY, &docs)?; + let max = got.iter().zip(&want).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + println!("{label:<12} {name:<8} {max:>12.6}"); + } + } + Ok(()) +} diff --git a/benchmark/workloads.rs b/benchmark/workloads.rs new file mode 100644 index 00000000..da5adaec --- /dev/null +++ b/benchmark/workloads.rs @@ -0,0 +1,107 @@ +//! Seven workloads of documents to embed and rerank, because they stress different things. A batch +//! of short docs is dominated by per-call overheads (tokenization, thread spawns), while 30 docs at +//! the 512-token truncation cap, recall's rerank worst case, are dominated by GEMM throughput and +//! memory behavior. The ragged batch is real indexing's shape: batch-longest padding masks most +//! attention columns, whose softmax weights underflow, and the scores×V GEMM then reads what they +//! leave behind. The real batch spreads its lengths the way measured chunks do, so it shows what a +//! backend spends on padding a mixed batch. The 256-document workloads are an indexing call's size +//! on the CPU, and show how grouping by length scales. The 100-document one is recall's pool at +//! `--candidates 100`. +//! +//! No two documents of a workload are the same text: the GPU embedder embeds a repeated text +//! once, so a workload with repeats would time fewer documents than it holds. + +/// The query every reranker workload scores its documents against. +pub const QUERY: &str = "why did we move the reranker off the onnx runtime"; + +fn short_docs() -> Vec { + [ + "the reranker is a cross-encoder scoring query-passage pairs jointly.", + "onnx runtime uses MLAS on the CPU, which on Apple Silicon runs on NEON, not AMX.", + "a hand-written forward calls Accelerate cblas_sgemm, which reaches the AMX matrix units.", + "the embedding model is bge-small-en-v1.5, a 384-dimensional BERT sentence encoder.", + "recall fuses vector ANN and BM25 hits by reciprocal rank before reranking.", + "the memory is a lance dataset with an IVF_PQ vector index and a full-text index.", + "candle's metal backend was missing a layer-norm kernel, so it could not run the model.", + "the cat knocked a glass off the counter and it shattered on the floor.", + "quarterly revenue rose twelve percent on strong subscription renewals.", + "tokenization uses the huggingface tokenizers crate loading the model's tokenizer.json.", + "softmax over the attention scores uses a vectorized exp from the platform seam.", + "the recipe needs two cups of flour, a teaspoon of salt, and three eggs.", + "fp8 has no hardware path on apple silicon, so int8 is the only accelerated low-precision.", + "attention masks let the transformer ignore padding tokens in a ragged batch.", + "the marathon route winds through six neighborhoods before the riverside finish.", + "hf-hub fetches whole files because the xet cdn taxes every byte-range read.", + ] + .map(String::from) + .to_vec() +} + +const SENT: &str = "recall fuses vector ann and bm25 hits by reciprocal rank before the cross-encoder \ + rescores each candidate against the query using joint attention over the pair. "; + +/// `n` documents of about 500 tokens, truncated at 512: recall's rerank candidates at the cap. +fn capped_docs(n: usize) -> Vec { + (0..n).map(|i| format!("{i} {}", SENT.repeat(18))).collect() +} + +/// A ragged batch, two ~512-token docs and the rest short: real indexing's shape, and the one that +/// exposes padding stalls. +fn mixed_docs() -> Vec { + let short = short_docs(); + let mut docs = capped_docs(2); + docs.extend((0..16 - docs.len()).map(|i| short[i % short.len()].clone())); + docs +} + +/// Words of `SENT` whose embedder token counts land on the 32 quantiles of real chunk lengths, +/// measured over 12,971 chunks of 100 Claude Code sessions: 21 up to the 512 cap, median 314. +const REAL_WORDS: [usize; 32] = [ + 13, 30, 38, 51, 66, 87, 106, 126, 145, 163, 176, 187, 196, 205, 213, 218, 226, 232, 238, 247, 255, 262, 269, 280, + 288, 296, 306, 314, 327, 340, 362, 363, +]; + +/// One doc per quantile of real chunk lengths, in a scrambled order: the length spread indexing +/// hands `embed`, and about recall's 30 rerank candidates. +pub fn real_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..32).map(|i| words[..REAL_WORDS[i * 13 % 32]].join(" ")).collect() +} + +/// Like `mixed_docs` but 256 documents, an indexing call's size on the CPU. +fn big_mixed_docs() -> Vec { + let short = short_docs(); + let mut docs = capped_docs(32); + docs.extend((0..256 - docs.len()).map(|i| format!("{i} {}", short[i % short.len()]))); + docs +} + +/// Like `real_docs` but 256 documents, with the same length spread: an indexing call's size on the +/// CPU. +fn big_real_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..256) + .map(|i| format!("{i} {}", words[..REAL_WORDS[i % 32]].join(" "))) + .collect() +} + +/// Like `real_docs` but 100 documents: recall's pool at `--candidates 100`. +fn deep_docs() -> Vec { + let words: Vec<&str> = SENT.split_whitespace().cycle().take(400).collect(); + (0..100) + .map(|i| format!("{i} {}", words[..REAL_WORDS[i * 13 % 32]].join(" "))) + .collect() +} + +/// Each workload's label and documents. +pub fn workloads() -> Vec<(&'static str, Vec)> { + vec![ + ("16×short", short_docs()), + ("30×~500tok", capped_docs(30)), + ("16×mixed", mixed_docs()), + ("32×real", real_docs()), + ("100×real", deep_docs()), + ("256×mixed", big_mixed_docs()), + ("256×real", big_real_docs()), + ] +} diff --git a/docs/automation.md b/docs/automation.md index f7eb3a7e..53bb5d8e 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -78,9 +78,10 @@ validates a producer's output without writing anything, so run it before wiring memory compete for its head on the Hub, and the one that loses retries. The one case that can leave rows out is a first publish to an empty memory, which has no head to check against. Setting up one host before the others avoids it. -- **Secrets held back.** funes redacts credentials at index time; on push, a separate always-on gate - withholds any chunk that still contains one — the clean rows publish, and the push exits non-zero - (code `2`) only if that leaves nothing to publish. Run `funes scrub`, then the next push includes it. +- **Secrets redacted on the way out.** The local memory keeps sessions as written, and on push an + always-on gate redacts credentials from what it publishes. A block it can't redact in place is + held back, the clean rows publish, and the push exits non-zero (code `2`) only if that leaves + nothing to publish. Run `funes scrub`, then the next push includes it. - **The card rides along.** A push to a memory at the repo root creates the repo's dataset card (tagged `funes`) and keeps its stats fresh — in the same commit as the data. A hand-written card is never touched. diff --git a/docs/configuration.md b/docs/configuration.md index 94b2b9b9..d6ddef1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,7 +88,7 @@ file-grained cache design and cold-versus-warm behavior. | `FUNES_BIN` | Binary path recorded in supported MCP registrations and used by the pi bridge. Hook workers instead find `funes` on `PATH` or in common install directories. | | `FUNES_MEMORY` | Per-run memory override understood by the pi extension; otherwise its copy of the host's binding is used. | | `FUNES_INTEGRATIONS` | Directory `funes add` installs an agent integration from, instead of the published one. funes can't vouch for those files, so it asks you to confirm before running them — once, until they change. | -| `FUNES_TRUFFLEHOG` | Explicit TruffleHog binary for secret scanning. Index-time redaction is best-effort; push and scrub scanning fail closed. | +| `FUNES_TRUFFLEHOG` | Explicit TruffleHog binary for secret scanning. Push and scrub refuse to run without one. | | `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN`, `HUGGINGFACE_TOKEN` | Hugging Face authentication, in the precedence shown above. | | `HF_TOKEN_PATH` | The token file, when it is not `$HF_HOME/token`. | | `HF_HUB_DISABLE_IMPLICIT_TOKEN` | Set to anything: no token is used, from the variables or a file. | diff --git a/docs/funes-jsonl.md b/docs/funes-jsonl.md index 34e66d42..e4597c26 100644 --- a/docs/funes-jsonl.md +++ b/docs/funes-jsonl.md @@ -125,8 +125,9 @@ Run it before you publish a producer. Behaviour, not contract — it may evolve; the identity rule above will not. - **Elides** inline base64 `data:` payloads to `[elided]`. -- **Redacts** secrets before chunking, best-effort; whatever slips through is caught by the fail-closed - gate on `push`, so a leaked token never reaches a published memory. +- **Redacts** secrets only on `push`: the local memory keeps them as written, and the fail-closed + gate on `push` redacts them from what it publishes, so a leaked token never reaches a published + memory. - **Renders** each block to the text that is embedded and full-text indexed: `text` and `thinking` as-is; `tool_use` as `[tool_use ] ` (`[tool_use None] ` without a name); `tool_result` as `[tool_result ] ` (`[tool_result] ` without a name). diff --git a/docs/push.md b/docs/push.md index e9394175..f497cc4a 100644 --- a/docs/push.md +++ b/docs/push.md @@ -50,15 +50,18 @@ run this at session boundaries automatically; see [automation.md](automation.md) ## Keeping secrets out: the gate and `funes scrub` -When TruffleHog is available, indexing redacts detected credentials before storing a session. That -first pass is best-effort: local indexing still works without the scanner because the local memory -has not crossed a publication boundary. It prints a warning that index-time redaction is disabled. +Indexing stores sessions as written, credentials included: the local memory is not a publication +boundary, so indexing doesn't scan it. Push is the hard boundary. A separate, **always-on, fail-closed gate** requires TruffleHog and scans the rows about to leave the machine. It reconstructs complete content blocks before scanning, so a -secret split across chunks cannot evade detection. If any chunk of a block contains a secret, every -chunk of that block is held back; unrelated clean rows still publish with a warning. Only when that -leaves *nothing* to publish does push exit non-zero (code `2`): +secret split across chunks cannot evade detection. A block that contains a secret is redacted the +way [`funes scrub`](#what-funes-scrub-changes) does it, and its redacted chunks are published in +place of the stored ones, which the local memory keeps as they are. A block that can't be redacted +in place is held back whole: a secret found in an encoded region, a redaction that still scans dirty, +or one that re-chunks into a different number of chunks than the block has. Unrelated clean rows +still publish with a warning. Only when that leaves *nothing* to publish does push exit non-zero +(code `2`): ```console $ funes push /funes-memory diff --git a/src/chunk.rs b/src/chunk.rs index 2d0b3f89..c23fa46b 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -253,8 +253,9 @@ pub(crate) fn reconstruct_blocks(chunks: &[Chunk]) -> Vec<(Vec, String)> /// Re-chunk a block's (already rendered) `text` after it changed — e.g. a secret was redacted out — /// reusing the block's identity (session, turn, block_idx, role, …) from `template`. Ids and /// `split_idx` are regenerated from the new split count, so they stay coordinate-stable. An empty -/// result means nothing indexable survived. -pub(crate) fn resplit(template: &Chunk, text: &str) -> Vec { +/// result means nothing indexable survived. The pieces carry `tool_name`, the name `text` renders, +/// which a redaction of the text may have changed. +pub(crate) fn resplit(template: &Chunk, text: &str, tool_name: Option) -> Vec { split(text) .into_iter() .enumerate() @@ -262,15 +263,15 @@ pub(crate) fn resplit(template: &Chunk, text: &str) -> Vec { id: cid(&template.session_id, &template.turn_uuid, template.block_idx, si as i64), text: piece, split_idx: si as i64, + tool_name: tool_name.clone(), ..(*template).clone() }) .collect() } /// Whether an index (with `include_thinking`) stores a block of this type: it isn't a dropped -/// thinking block. Shared with redaction, so the blocks scanned for secrets are exactly the blocks -/// stored. -pub fn block_selected(block_type: &str, include_thinking: bool) -> bool { +/// thinking block. +fn block_selected(block_type: &str, include_thinking: bool) -> bool { block_type != "thinking" || include_thinking } @@ -485,7 +486,7 @@ mod tests { let fresh = chunks_from_turns(std::slice::from_ref(&t), true); let template = fresh[0].clone(); let rendered = render("text", &long, &None); - let re = resplit(&template, &rendered); + let re = resplit(&template, &rendered, None); assert_eq!(re.len(), fresh.len()); for (a, b) in re.iter().zip(&fresh) { assert_eq!(a.id, b.id); diff --git a/src/commands/index.rs b/src/commands/index.rs index 8d5d413a..a629381d 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -15,7 +15,6 @@ use crate::inference::{self, embed_batched, Embedder}; use crate::memory::dataset::{self, build_batch, model}; use crate::memory::lock; use crate::memory::Memory; -use crate::scan; use crate::traces::spool; use crate::traces::{self, repo, source}; use crate::ui::prompt::confirm; @@ -162,10 +161,9 @@ pub async fn ensure_local_memory(model: inference::EmbeddingModel) -> Result<()> } } -/// Elide every block's inline base64 `data:` URI payloads, before [`redact_units`] scans them: a -/// screenshot's entropy trips detectors on values that are not secrets, and excising one of those -/// plants a marker mid-payload that strands the rest of it in the store. Runs whether or not a -/// scanner is installed — the payload is unrecallable either way. +/// Elide every block's inline base64 `data:` URI payloads before they are stored: a payload is +/// unrecallable, and a screenshot's entropy trips the push gate's detectors on values that are not +/// secrets. fn elide_turns(turns: &mut [traces::Turn]) { for b in turns.iter_mut().flat_map(|t| t.blocks.iter_mut()) { if let Cow::Owned(elided) = chunk::elide_data_uris(&b.text) { @@ -174,82 +172,15 @@ fn elide_turns(turns: &mut [traces::Turn]) { } } -/// Redact secrets from `units`' turns *before* chunking — so a long key that chunking would split -/// across pieces is whole when scanned, and never reaches the embedding, the local memory, or (via -/// push) the Hub. One scanner run covers every unit given: the spawn costs ~1 s, the scan itself -/// milliseconds. Scans exactly the blocks the memory stores ([`chunk::block_selected`]). -/// Best-effort: removes a secret whose value byte-matches the stored text (the common case, real -/// newlines); anything that resists is caught downstream by the fail-closed push gate. Reports to -/// stderr what it removed, per unit. -fn redact_units( - units: &mut [&mut [traces::Turn]], - scanner: &dyn scan::SecretScanner, - include_thinking: bool, -) -> Result<()> { - let n_units = units.len(); - let mut blocks: Vec<(usize, &mut traces::Block)> = Vec::new(); - for (u, turns) in units.iter_mut().enumerate() { - for b in turns.iter_mut().flat_map(|t| t.blocks.iter_mut()) { - if chunk::block_selected(&b.block_type, include_thinking) { - blocks.push((u, b)); - } - } - } - if blocks.is_empty() { - return Ok(()); - } - let per_block = { - let texts: Vec<&str> = blocks.iter().map(|(_, b)| b.text.as_str()).collect(); - scan::scan_blocks(&texts, scanner)? - }; - let mut removed: Vec> = (0..n_units).map(|_| Vec::new()).collect(); - for ((u, b), findings) in blocks.into_iter().zip(&per_block) { - let r = scan::excise(&b.text, findings); - removed[u].extend(r.removed_detectors); - b.text = r.text; - } - for (turns, removed) in units.iter().zip(removed) { - if removed.is_empty() { - continue; - } - let sid = turns.first().map(|t| t.session_id.as_str()).unwrap_or("?"); - eprintln!( - " redacted {} secret(s) in {sid}: {}", - removed.len(), - scan::summary(removed.iter().map(String::as_str)) - ); - } - Ok(()) -} +/// Units a batch consumes, at most: a batch's units are read, written and recorded together. +const BATCH_UNITS: usize = 32; -/// Make a batch of units' turns what the memory stores — data URIs elided and, with a `scanner`, -/// secrets redacted, the batch in one scanner run — before any of them is chunked, since both -/// change the text a block splits into. -fn sanitize_units( - units: &mut [&mut [traces::Turn]], - include_thinking: bool, - scanner: Option<&impl scan::SecretScanner>, -) -> Result<()> { - for turns in units.iter_mut() { - elide_turns(turns); - } - if let Some(scanner) = scanner { - redact_units(units, scanner, include_thinking)?; - } - Ok(()) -} - -/// Units a batch consumes: those read are scanned for secrets in one scanner spawn (~1 s) rather -/// than one per unit, so a directory of hundreds of session files is scanned in seconds, not -/// minutes. -const SCAN_BATCH: usize = 32; - -/// Block text a batch holds before it is scanned: bounds the memory a batch of bulk units (a Hub +/// Block text a batch holds before it is written: bounds the memory a batch of bulk units (a Hub /// shard holds thousands of sessions) takes, where the unit count alone would not. -const SCAN_BATCH_BYTES: usize = 64 << 20; +const BATCH_BYTES: usize = 64 << 20; -/// Read units from the front of `units` until a batch fills — [`SCAN_BATCH`] consumed, or -/// [`SCAN_BATCH_BYTES`] of block text read, whichever comes first. `read(n, unit)` gives the +/// Read units from the front of `units` until a batch fills: [`BATCH_UNITS`] consumed, or +/// [`BATCH_BYTES`] of block text read, whichever comes first. `read(n, unit)` gives the /// `n`th unit's turns, or `None` for one the batch has no use for (already indexed, rejected). /// Either way the unit is consumed and counts, so whatever a backlog holds, a caller's check /// between batches is never more than a batch away. Returns one entry per unit consumed, in @@ -270,25 +201,13 @@ fn take_batch( .sum::(); } batch.push(turns); - if batch.len() >= SCAN_BATCH || bytes >= SCAN_BATCH_BYTES { + if batch.len() >= BATCH_UNITS || bytes >= BATCH_BYTES { break; } } Ok(batch) } -/// The secret scanner, if installed. Best-effort: without one, indexing continues unredacted — the -/// push gate still scans, fail-closed, before any upload, so a secret can't reach the Hub. -fn find_scanner() -> Option { - match scan::Trufflehog::find() { - Ok(s) => Some(s), - Err(e) => { - eprintln!("note: secret redaction disabled — {e}"); - None - } - } -} - /// A unit's distinct-session count and a log label: `" ()"` for a single session (a /// JSONL file), `" sessions"` for a bulk unit (many sessions in one artifact), and the unit's `key` (its /// path) when it has no turns at all. The borrow of `turns` is confined here so callers keep it mutable. @@ -458,17 +377,16 @@ pub(crate) fn local_index_coverage() -> Option { }) } -/// A set-up indexer: it holds the memory lock, embedder, dataset, redaction scanner, and incremental -/// state, so a caller can index units in whatever batches it likes — one at a time to check the -/// clock between them, or all at once — without reloading the model. Build the indexes once at the -/// end with [`Indexer::finalize`]. +/// A set-up indexer: it holds the memory lock, embedder, dataset, and incremental state, so a +/// caller can index units in whatever batches it likes (one at a time to check the clock between +/// them, or all at once) without reloading the model. Build the indexes once at the end with +/// [`Indexer::finalize`]. struct Indexer { uri: String, ds: Option, /// [`stored_ids`] at open plus everything written this run — the dedup baseline for new chunks. existing: HashSet, embedder: Box, - scanner: Option, include_thinking: bool, /// cwd → resolved `repo` value, so each distinct checkout runs `git` once across the run. repo_cache: HashMap, @@ -497,6 +415,11 @@ struct Indexer { /// Units (by index) whose read failed this run — reported once. Signed units also keep their /// refusal in state until their content or this build changes. rejected: HashSet, + /// A unit's state was recorded since the last [`Indexer::persist`]. + recorded: bool, + /// The spool files this batch indexed, as key, signature and progress label, to drain once + /// [`Indexer::persist`] has recorded them. + to_drain: Vec<(String, String, String)>, } /// Enumerate every source's units (each source orders its own — recency-desc), tagged with the @@ -532,7 +455,7 @@ fn collect_units(sources: &[Box]) -> Result>, no_thinking: bool) -> Result { std::fs::create_dir_all(home::dir())?; let interactive = std::io::stdin().is_terminal(); @@ -571,7 +494,6 @@ impl Indexer { }; let embedder: Box = inference::embedder(model)?; - let scanner = find_scanner(); let units = collect_units(&sources)?; if !no_thinking { @@ -593,7 +515,6 @@ impl Indexer { ds, existing, embedder, - scanner, include_thinking: !no_thinking, repo_cache: HashMap::new(), state, @@ -611,6 +532,8 @@ impl Indexer { n_embedded: 0, languages, rejected: HashSet::new(), + recorded: false, + to_drain: Vec::new(), }) } @@ -637,9 +560,18 @@ impl Indexer { .collect() } - /// Record a unit's state and persist it, so an interrupted run resumes where it stopped. - fn record(&mut self, key: &str, state: UnitState) -> Result<()> { + /// Record a unit's state, unwritten until the next [`Indexer::persist`]. + fn record(&mut self, key: &str, state: UnitState) { self.state.insert(key.to_string(), state); + self.recorded = true; + } + + /// Write every unit's state, if one was recorded since the last write, so an interrupted run + /// resumes where it stopped. + fn persist(&mut self) -> Result<()> { + if !std::mem::take(&mut self.recorded) { + return Ok(()); + } std::fs::write(&self.state_path, serde_json::to_string_pretty(&self.state)?)?; write_index_coverage(&self.coverage_path, &self.sources, &self.units, &self.state) } @@ -676,7 +608,7 @@ impl Indexer { level: None, refused: Some(VERSION.to_string()), }, - )?; + ); } Ok(None) } @@ -684,22 +616,24 @@ impl Indexer { } } - /// Read and sanitize a batch of units, with one scanner run. Returns one entry per unit - /// consumed: its turns, or `None` when skipped or refused. + /// Read a batch of units, their data URIs elided. Returns one entry per unit consumed: its turns, + /// or `None` when skipped or refused. fn read_batch( &mut self, units: &[usize], label: impl Fn(usize) -> String, ) -> Result>>> { let mut read = take_batch(units, |n, &i| self.read_unit(i, &label(n)))?; - let mut all: Vec<&mut [traces::Turn]> = read.iter_mut().flatten().map(Vec::as_mut_slice).collect(); - sanitize_units(&mut all, self.include_thinking, self.scanner.as_ref())?; + for turns in read.iter_mut().flatten() { + elide_turns(turns); + } Ok(read) } - /// Write all tiers of unit `i` from sanitized `turns`, keeping only new chunks in one append. - /// Record that its rows are in the memory before draining its spool file; the memory itself - /// carries any embeddings still owed. A bulk source stays one fragment per unit. + /// Write all tiers of unit `i` from its `turns`, keeping only new chunks in one append, and + /// record that its rows are in the memory. A spool file left with nothing owed is queued to + /// drain. The memory itself carries any embeddings still owed. A bulk source stays one fragment + /// per unit. async fn write_unit(&mut self, i: usize, turns: &[traces::Turn], progress: &str) -> Result { let (key, sig) = { let unit = &self.units[i].1; @@ -725,12 +659,21 @@ impl Indexer { eprintln!("{progress} {label} — no indexable content"); 0 } else { - // A unit can carry one id twice (a turn re-emitted under its `turn_uuid`); the first wins, - // as it would have had the two arrived in separate runs. + // A block stored once takes no more pieces: a re-read can split it into more than the + // memory holds (an edit re-emitted under its `turn_uuid`, or a block an older funes stored + // with its secrets cut out), and the extra ones would sit behind the old ones. A block's + // first piece stands for it, since its pieces are written together, and the first + // occurrence of a block in a unit wins, as it would have in separate runs. let mut in_batch = HashSet::new(); + let mut new_block = false; let new_chunks: Vec = chunks .into_iter() - .filter(|c| !self.existing.contains(&c.id) && in_batch.insert(c.id.clone())) + .filter(|c| { + if c.split_idx == 0 { + new_block = !self.existing.contains(&c.id) && in_batch.insert(c.id.clone()); + } + new_block + }) .collect(); if new_chunks.is_empty() { eprintln!("{progress} {label} — {total_chunks} chunks, all already indexed"); @@ -751,11 +694,9 @@ impl Indexer { level: Some(Tier::ToolResult), refused: None, }, - )?; + ); if drains(&key, Tier::ToolResult, self.include_thinking) { - if let Err(e) = drain(&key, sig) { - eprintln!("{progress} {key} — indexed, but the spool copy stayed: {e:#}"); - } + self.to_drain.push((key, sig.clone(), progress.to_string())); } } self.n_sessions += sessions; @@ -774,6 +715,14 @@ impl Indexer { self.write_unit(units[n], &turns, &label(n)).await?; } } + // One state write per batch, since each write holds every unit's state. A spool file is + // drained only once the state says its unit is indexed. + self.persist()?; + for (key, sig, progress) in std::mem::take(&mut self.to_drain) { + if let Err(e) = drain(&key, &sig) { + eprintln!("{progress} {key} — indexed, but the spool copy stayed: {e:#}"); + } + } Ok(consumed) } @@ -1190,7 +1139,6 @@ pub fn check(path: &Path, no_thinking: bool, limit: Option) -> Result) -> Result = read.iter_mut().flatten().map(Vec::as_mut_slice).collect(); - sanitize_units(&mut all, !no_thinking, scanner.as_ref())?; + for unit_turns in read.iter_mut().flatten() { + elide_turns(unit_turns); + } let mut rejections = rejections.into_iter(); for (n, unit_turns) in read.iter().enumerate() { let Some(unit_turns) = unit_turns else { @@ -1254,6 +1203,47 @@ pub async fn run_index(path: &Path, no_thinking: bool, max_sessions: Option) -> Result { + let mut indexer = Indexer::open(vec![source::open(path, max_sessions)?], false).await?; + let total = indexer.unit_count(); + let all: Vec = (0..total).collect(); + let t0 = Instant::now(); + let mut done = 0; + while done < total { + done += indexer.write_units(&all[done..], done, total).await?; + } + let write = t0.elapsed(); + let fragments = match &indexer.ds { + Some(ds) => dataset::fragments_to_compact(ds).await?, + None => 0, + }; + let (sessions, chunks) = (indexer.n_sessions, indexer.n_chunks); + indexer.work_remaining = true; + let t1 = Instant::now(); + indexer.finalize().await?; + Ok(RowsWritten { + sessions, + chunks, + fragments, + write, + finalize: t1.elapsed(), + }) +} + /// A first interactive index estimated at ≥ this many seconds prompts before continuing. const FIRST_INDEX_PROMPT_SECS: u64 = 120; @@ -1724,23 +1714,7 @@ mod tests { } #[test] - fn sanitize_elides_payloads_before_the_scanner_sees_them() { - // Records what it is shown, and finds the key wherever it is. - struct Recorder(std::cell::RefCell); - impl scan::SecretScanner for Recorder { - fn scan(&self, texts: &[&str]) -> Result>> { - self.0.borrow_mut().push_str(&texts.join("\n")); - let key = scan::Finding { - detector: "PrivateKey".into(), - raw: "TOPSECRET".into(), - decoder: "PLAIN".into(), - }; - Ok(texts - .iter() - .map(|t| t.contains("TOPSECRET").then(|| key.clone()).into_iter().collect()) - .collect()) - } - } + fn a_stored_block_loses_its_data_uris_and_keeps_its_text_as_written() { let mut turns = vec![traces::Turn { format: traces::FORMAT_VERSION, session_id: "sess".into(), @@ -1760,81 +1734,10 @@ mod tests { source_path: String::new(), harness: "codex".into(), }]; - let scanner = Recorder(std::cell::RefCell::new(String::new())); - sanitize_units(&mut [&mut turns], true, Some(&scanner)).unwrap(); - assert_eq!( - turns[0].blocks[0].text, - r#"{"image_url":"data:image/png;base64,[elided]"} key=[REDACTED:PrivateKey]"# - ); - assert!( - !scanner.0.borrow().contains("iVBORw0KGgo"), - "the scanner must never see the payload: excising a match inside it would strand the rest" - ); - } - - #[test] - fn redact_scans_every_unit_in_one_pass_and_only_the_blocks_stored() { - // Finds the secret a text carries, so a finding must land on the unit it came from. - struct Counting(std::cell::Cell); - impl scan::SecretScanner for Counting { - fn scan(&self, texts: &[&str]) -> Result>> { - self.0.set(self.0.get() + 1); - let finding = |detector: &str, raw: &str| scan::Finding { - detector: detector.into(), - raw: raw.into(), - decoder: "PLAIN".into(), - }; - Ok(texts - .iter() - .map(|t| { - let mut found = Vec::new(); - if t.contains("SECRET") { - found.push(finding("PrivateKey", "SECRET")); - } - if t.contains("TOKEN") { - found.push(finding("Slack", "TOKEN")); - } - found - }) - .collect()) - } - } - let block = |bt: &str, text: &str| traces::Block { - block_type: bt.into(), - text: text.into(), - tool_name: None, - tool_use_id: None, - }; - let turn = |blocks: Vec| traces::Turn { - format: traces::FORMAT_VERSION, - session_id: "sess".into(), - cwd: None, - workdir: "proj".into(), - turn_uuid: "turn".into(), - parent_uuid: None, - seq: 0, - ts: String::new(), - role: "user".into(), - blocks, - source_path: String::new(), - harness: "claude_code".into(), - }; - let mut a = vec![turn(vec![ - block("text", "note SECRET here"), - block("thinking", "SECRET thought"), - ])]; - let mut b = vec![turn(vec![block("tool_result", "output TOKEN dump")])]; - let scanner = Counting(std::cell::Cell::new(0)); - redact_units(&mut [&mut a, &mut b], &scanner, false).unwrap(); - assert_eq!(scanner.0.get(), 1, "one scanner run for both units"); - assert_eq!(a[0].blocks[0].text, "note [REDACTED:PrivateKey] here"); - assert_eq!( - a[0].blocks[1].text, "SECRET thought", - "a thinking block --no-thinking won't store is not scanned" - ); + elide_turns(&mut turns); assert_eq!( - b[0].blocks[0].text, "output [REDACTED:Slack] dump", - "each unit of the batch gets its own secrets, and only those" + turns[0].blocks[0].text, r#"{"image_url":"data:image/png;base64,[elided]"} key=TOPSECRET"#, + "the payload goes, and the push gate is what redacts a secret" ); } @@ -1865,20 +1768,20 @@ mod tests { // Tiny units fill a batch by count; the rest go in the next. let batch = take_batch(&units, |_, _| Ok(Some(turns_of(1)))).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); + assert_eq!(batch.len(), BATCH_UNITS); let rest = take_batch(&units[batch.len()..], |_, _| Ok(Some(turns_of(1)))).unwrap(); - assert_eq!(rest.len(), 40 - SCAN_BATCH); + assert_eq!(rest.len(), 40 - BATCH_UNITS); // A unit the pass has no use for is consumed and counts: a backlog of them still ends a // batch, so a budget check is never more than a batch away. let batch = take_batch(&units, |n, _| Ok((n % 2 == 0).then(|| turns_of(1)))).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); - assert_eq!(batch.iter().flatten().count(), SCAN_BATCH / 2); + assert_eq!(batch.len(), BATCH_UNITS); + assert_eq!(batch.iter().flatten().count(), BATCH_UNITS / 2); let batch = take_batch(&units, |_, _| Ok(None)).unwrap(); - assert_eq!(batch.len(), SCAN_BATCH); + assert_eq!(batch.len(), BATCH_UNITS); // Text fills a batch before the count does. - let batch = take_batch(&units, |_, _| Ok(Some(turns_of(SCAN_BATCH_BYTES / 4)))).unwrap(); + let batch = take_batch(&units, |_, _| Ok(Some(turns_of(BATCH_BYTES / 4)))).unwrap(); assert_eq!(batch.len(), 4); } diff --git a/src/commands/push.rs b/src/commands/push.rs index 9720e418..6ea50d3b 100644 --- a/src/commands/push.rs +++ b/src/commands/push.rs @@ -2,9 +2,9 @@ //! //! Streamed, never a full mirror. "What's already there" is the memory's own chunk ids, so the //! delta is `embedded_local_ids − remote_ids`. `push` is orchestration: -//! it computes the delta, holds back any row that still contains a secret (redaction happens at -//! index time; this is the egress backstop — the rows wait for `funes scrub`), and drives the HF -//! write operations in [`crate::memory::remote`], which own the atomic, parent-commit-guarded commits. +//! it computes the delta, redacts the secrets in the rows it ships (holding back the blocks it +//! can't redact in place, which wait for `funes scrub`), and drives the HF write operations in +//! [`crate::memory::remote`], which own the atomic, parent-commit-guarded commits. //! //! - **First publish:** build the dataset locally (data + FTS/IVF indexes) and upload every file in //! one commit. @@ -22,6 +22,7 @@ use super::compact; use crate::hub; +use crate::inference::{self, embed_batched, EmbeddingModel}; use crate::memory::card::{self, CardAction, CardCtx}; use crate::memory::dataset::{self, RecordBatch}; use crate::memory::lock; @@ -86,8 +87,8 @@ async fn held_among(local: &Dataset, pending: &HashSet) -> Option 0).then_some(skipped) + let held = gate(&rows).ok()?.held; + (held.rows > 0).then_some(held) } /// Every embedded row of the local memory, whole — a publish ships each row as it stands. @@ -401,10 +402,11 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: return Ok(format!("{}: up to date ({} chunks)\n{note}", target.label(), remote_ids.len()).into()); } - // 4. Rows, then hold back any that still contain a secret. Re-stamp each batch with the local + // 4. Rows, then redact the secrets they hold. Re-stamp each batch with the local // dataset's schema so its metadata (the embedding-model id) rides along — scan-result batches // drop it, and on first publish that schema is what the new dataset persists. let schema = dataset::schema_of(&local); + let model = dataset::embedding_model(&local)?; // Only a first publish with nothing named ships the whole memory; anything else, `to_push` // is the answer — including a first publish of a selection. let rows = if first_publish && sessions.is_empty() { @@ -414,21 +416,29 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: }; let batches = dataset::restamped(&schema, rows)?; - // Drop any row whose text still holds a secret — hold it back from the Hub rather than block the - // whole push. `funes scrub` redacts it in the local memory; the next push then ships it. + // Redact the secrets the rows hold, and hold back the blocks that can't be redacted in place + // rather than block the whole push: `funes scrub` redacts or drops them in the local memory, + // and the next push ships what it leaves. let n_scanning: usize = batches.iter().map(|b| b.num_rows()).sum(); eprintln!("scanning {n_scanning} chunk(s) for secrets…"); - let (batches, skipped) = drop_secret_rows(batches)?; + let gated = gate(&batches)?; + let batches = gated_rows(batches, &gated, model, |texts| { + eprintln!("re-embedding {} redacted chunk(s)…", texts.len()); + let mut embedder = inference::embedder(model)?; + embed_batched(embedder.as_mut(), texts, |_| {}) + })?; + let batches = dataset::restamped(&schema, batches)?; + let Gated { held, redacted, .. } = gated; let n_chunks: usize = batches.iter().map(|b| b.num_rows()).sum(); - if n_chunks == 0 && skipped.rows > 0 { + if n_chunks == 0 && held.rows > 0 { // Everything was held back: nothing reached the Hub. Mark `blocked` so the CLI exits non-zero // — automation must not read this as a successful publish. return Ok(Pushed { report: format!( "{}: nothing published — held back {} row(s) with secrets ({}); run `funes scrub`, then push again\n", target.label(), - skipped.rows, - skipped.summary + held.rows, + held.summary ), blocked: true, }); @@ -441,7 +451,6 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: // (and two stores sharing a repo would fight over one card's stats) — under a prefix it's // the owner's. The chunk count is the post-push total — best-effort: rows another writer // lands concurrently are missed until the next push refreshes the card. - let model = dataset::embedding_model(&local)?; let date = Utc::now().format("%Y-%m-%d").to_string(); let ctx = CardCtx { repo: &w.repo_id, @@ -485,9 +494,10 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: n => format!("pushed {n} chunks"), }; return Ok(format!( - "{}: {pushed} (commit {oid})\n{card_note}{}", + "{}: {pushed} (commit {oid})\n{card_note}{}{}", target.label(), - skipped.warning() + redacted.note(), + held.warning() ) .into()); } @@ -539,7 +549,8 @@ pub async fn run_push(target: Memory, compact: bool, confirm: Confirm, sessions: eprintln!("compacting the remote…"); out.push_str(&compact_auto(&w).await); } - out.push_str(&skipped.warning()); + out.push_str(&redacted.note()); + out.push_str(&held.warning()); Ok(out.into()) } @@ -564,7 +575,8 @@ fn card_file(remote_readme: Result>, ctx: &CardCtx) -> (Option) -> Result<(Vec, Skipped)> { +/// Blocks a push redacted secrets from before publishing them. +#[derive(Default)] +struct Redacted { + blocks: usize, + /// Detectors that fired, e.g. `PrivateKey×1, AWS×2`. + summary: String, +} + +impl Redacted { + /// A line for the push output, or empty if nothing was redacted. + fn note(&self) -> String { + if self.blocks == 0 { + return String::new(); + } + format!( + " redacted secrets in {} block(s) before publishing ({})\n", + self.blocks, self.summary + ) + } +} + +/// What the secret gate makes of the rows a push would upload. +struct Gated { + /// Per row, across the batches in order: its block held a secret, so the row is replaced or held + /// back. + dirty: Vec, + /// The redacted blocks' pieces, each carrying the id of the row it replaces. + replacements: Vec, + redacted: Redacted, + held: Skipped, +} + +/// Scan the to-push `batches` for secrets, a whole block at a time: a block's chunks are +/// reconstructed into their contiguous text (so a secret `split` cut across chunks is whole and +/// detectable), and every block is scanned in one pass. A block that holds a secret is redacted as +/// `funes scrub` does it: the secret excised from its text, the text re-split by the same chunker, +/// the result scanned again. Its pieces replace its rows only if they carry the same ids, the +/// block's coordinates, so the remote holds the ids the local memory does. Otherwise the whole +/// block is held back, its other chunks carrying the rest of the secret: a secret the scanner +/// decoded from an encoded region, a redaction that still scans dirty, one re-split into another +/// number of pieces. Fail-closed on the scanner: a push must scan before it uploads. +fn gate(batches: &[RecordBatch]) -> Result { + let mut gated = Gated { + dirty: Vec::new(), + replacements: Vec::new(), + redacted: Redacted::default(), + held: Skipped::default(), + }; // Nothing to scan needs no scanner. if batches.iter().all(|b| b.num_rows() == 0) { - return Ok(( - batches, - Skipped { - rows: 0, - summary: String::new(), - }, - )); + return Ok(gated); } let scanner = scan::Trufflehog::find()?; // Row order across batches matches `chunks_from_batches`, so a chunk's index is its global row. - let chunks = dataset::chunks(&batches); + let chunks = dataset::chunks(batches); let blocks = chunk::reconstruct_blocks(&chunks); let texts: Vec<&str> = blocks.iter().map(|(_, text)| text.as_str()).collect(); let found = scan::scan_blocks(&texts, &scanner)?; + gated.dirty = vec![false; chunks.len()]; - let mut dirty = vec![false; chunks.len()]; - let mut detectors: Vec = Vec::new(); - for ((idxs, _), findings) in blocks.iter().zip(&found) { + // One tally per (block, distinct detector), so a summary reads "PrivateKey×". + let (mut redacted, mut held) = (Vec::new(), Vec::new()); + let mut candidates = Vec::new(); + for ((idxs, text), findings) in blocks.iter().zip(&found) { if findings.is_empty() { continue; } for &i in idxs { - dirty[i] = true; + gated.dirty[i] = true; + } + let r = scan::excise(text, findings); + let tool_name = chunks[idxs[0]] + .tool_name + .as_deref() + .map(|n| scan::excise(n, findings).text); + let pieces = chunk::resplit(&chunks[idxs[0]], &r.text, tool_name); + let same_ids = pieces.iter().map(|p| &p.id).eq(idxs.iter().map(|&i| &chunks[i].id)); + if r.fully_redacted && same_ids { + candidates.push((idxs, findings, r.text, pieces)); + } else { + gated.held.rows += idxs.len(); + held.extend(scan::detectors(findings)); } - // One tally per (block, distinct detector), so the warning reads "PrivateKey×". - detectors.extend(scan::detectors(findings)); } - let dropped = dirty.iter().filter(|&&d| d).count(); - if dropped == 0 { - return Ok(( - batches, - Skipped { - rows: 0, - summary: String::new(), - }, - )); + // Excising one match can expose another (removing one from a long base64 run re-aligns the + // window the next is found in), so a redaction stands only if it scans clean. + if !candidates.is_empty() { + let rescan: Vec<&str> = candidates.iter().map(|(_, _, text, _)| text.as_str()).collect(); + let left = scan::scan_blocks(&rescan, &scanner)?; + for ((idxs, findings, _, pieces), left) in candidates.into_iter().zip(&left) { + if left.is_empty() { + gated.replacements.extend(pieces); + gated.redacted.blocks += 1; + redacted.extend(scan::detectors(findings)); + } else { + gated.held.rows += idxs.len(); + held.extend(scan::detectors(findings)); + } + } } + gated.redacted.summary = scan::summary(redacted.iter().map(String::as_str)); + gated.held.summary = scan::summary(held.iter().map(String::as_str)); + Ok(gated) +} - // Drop the dirty rows batch by batch, mapping each global row index back via a running offset. - let mut clean = Vec::with_capacity(batches.len()); +/// The rows a push uploads once gated: every row outside a dirty block as it stands, then the +/// redacted blocks' pieces under their rows' ids, embedded by `embed`, which only runs when a block +/// was redacted. The rows of a held-back block stay behind. +fn gated_rows( + batches: Vec, + gated: &Gated, + model: EmbeddingModel, + embed: impl FnOnce(&[&str]) -> Result>>, +) -> Result> { + if !gated.dirty.contains(&true) { + return Ok(batches); + } + // Keep the rows of clean blocks batch by batch, mapping each global row index back via a running + // offset. + let mut out = Vec::with_capacity(batches.len() + 1); let mut base = 0usize; for b in &batches { - clean.push(dataset::keep_rows(b, |i| !dirty[base + i])?); + out.push(dataset::keep_rows(b, |i| !gated.dirty[base + i])?); base += b.num_rows(); } - let summary = scan::summary(detectors.iter().map(String::as_str)); - Ok((clean, Skipped { rows: dropped, summary })) + if !gated.replacements.is_empty() { + let texts: Vec<&str> = gated.replacements.iter().map(|c| c.text.as_str()).collect(); + let vectors = embed(&texts)?; + out.push(dataset::build_batch(model, &gated.replacements, Some(&vectors))?); + } + Ok(out) } /// Best-effort compaction during a normal push: one attempt, never retried. The data is already @@ -844,12 +930,81 @@ mod tests { ) } + /// Zero vectors in place of an embedder, for the rows a gate redacted. + fn zero_vectors(texts: &[&str]) -> Result>> { + Ok(vec![vec![0.0f32; dataset::DIM as usize]; texts.len()]) + } + #[test] - fn drop_secret_rows_holds_back_an_escaped_key() { - // The exact shape that leaked: a key stored with escaped `\n` (literal backslash-n), as a - // JSON-encoded transcript or a logged blob would hold it. trufflehog still detects it, but - // its canonical `raw` (real newlines) is not a substring of the stored bytes — value - // matching missed it and pushed it. Line-based location must hold it back. + fn a_redacted_block_ships_under_its_rows_ids() { + if scan::Trufflehog::find().is_err() { + eprintln!("skip: trufflehog not found"); + return; + } + let Some(key) = keygen(&["-t", "ed25519"]) else { + eprintln!("skip: ssh-keygen unavailable"); + return; + }; + let (b, chunks) = batch(&[turn(0, "clean note"), turn(1, &format!("deploy key: {key}"))]); + assert_eq!(chunks.len(), 2, "precondition: one chunk per turn"); + let ids = dataset::ids_in(std::slice::from_ref(&b)); + + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!( + gated.held.rows, 0, + "a key whose redaction keeps the block's pieces is redacted, not held" + ); + assert_eq!(gated.redacted.blocks, 1); + assert!( + gated.redacted.summary.contains("PrivateKey"), + "summary: {}", + gated.redacted.summary + ); + + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + assert_eq!(dataset::ids_in(&shipped), ids, "the redacted row keeps its id"); + let row = dataset::chunks(&shipped) + .into_iter() + .find(|c| c.id == chunks[1].id) + .unwrap(); + assert!(row.text.contains("[REDACTED:PrivateKey]"), "text: {}", row.text); + let body = key.lines().nth(1).unwrap(); + assert!(!row.text.contains(body), "no line of the key ships"); + } + + #[test] + fn a_secret_in_a_tool_name_ships_in_neither_the_text_nor_the_column() { + if scan::Trufflehog::find().is_err() { + eprintln!("skip: trufflehog not found"); + return; + } + let Some(key) = keygen(&["-t", "ed25519"]) else { + eprintln!("skip: ssh-keygen unavailable"); + return; + }; + // The block's text renders its tool name, which is also stored in a column of its own. + let mut call = turn(1, "{}"); + call.blocks[0].block_type = "tool_use".into(); + call.blocks[0].tool_name = Some(key.clone()); + let (b, _) = batch(&[turn(0, "clean note"), call]); + + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!(gated.redacted.blocks, 1, "the block is redacted in place"); + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + let body = key.lines().nth(1).unwrap(); + for row in dataset::chunks(&shipped) { + assert!(!row.text.contains(body), "text: {}", row.text); + let name = row.tool_name.unwrap_or_default(); + assert!(!name.contains(body), "tool_name: {name}"); + } + } + + #[test] + fn an_escaped_key_is_redacted_not_pushed() { + // A key stored with escaped `\n` (literal backslash-n), as a JSON-encoded transcript or a + // logged blob would hold it. trufflehog detects it, but its canonical `raw` (real newlines) + // is not a substring of the stored bytes, so the gate must find its escaped form to redact + // it. if scan::Trufflehog::find().is_err() { eprintln!("skip: trufflehog not found"); return; @@ -862,14 +1017,14 @@ mod tests { assert!(!escaped.contains('\n'), "precondition: no real newlines remain"); let (b, _) = batch(&[turn(0, "clean note"), turn(1, &format!("deploy key: {escaped}"))]); - let (clean, skipped) = drop_secret_rows(vec![b]).unwrap(); - assert_eq!(skipped.rows, 1, "the escaped key must be held back, not pushed"); - assert!(skipped.summary.contains("PrivateKey"), "summary: {}", skipped.summary); - assert_eq!( - clean.iter().map(|b| b.num_rows()).sum::(), - 1, - "the clean row stays" - ); + let gated = gate(std::slice::from_ref(&b)).unwrap(); + assert_eq!(gated.held.rows, 0, "the escaped key is redacted, not held"); + let shipped = gated_rows(vec![b], &gated, dataset::model::ENGLISH, zero_vectors).unwrap(); + assert_eq!(shipped.iter().map(|b| b.num_rows()).sum::(), 2, "both rows ship"); + let body = key.lines().nth(1).unwrap(); + for row in dataset::chunks(&shipped) { + assert!(!row.text.contains(body), "no line of the key ships: {}", row.text); + } } /// A two-session local dataset for the selection tests. @@ -960,12 +1115,15 @@ mod tests { eprintln!("skip: trufflehog not found"); return; } - let Some(key) = keygen(&["-t", "ed25519"]) else { + // A key that is the whole of a block spanning several chunks: its redaction re-splits into + // fewer pieces, so a push holds the block back rather than redact it in place. + let Some(key) = keygen(&["-t", "rsa", "-b", "4096"]) else { eprintln!("skip: ssh-keygen unavailable"); return; }; let dir = tempfile::tempdir().unwrap(); let (b, _) = batch(&[turn_sess("clean", 0, "notes on parsers"), turn_sess("dirty", 1, &key)]); + assert!(b.num_rows() > 2, "the secret must span multiple chunks"); let uri = dataset::table_uri(dir.path()); let schema = b.schema(); let reader = RecordBatchIterator::new(vec![Ok(b)], schema); @@ -979,7 +1137,7 @@ mod tests { let held = held_among(&ds, &all) .await .expect("the dirty pending block should be reported"); - assert!(held.rows >= 1); + assert_eq!(held.rows, by_session["dirty"].len()); assert!(held.summary.contains("PrivateKey"), "summary: {}", held.summary); let clean_only: HashSet = by_session["clean"].iter().cloned().collect(); @@ -1023,14 +1181,24 @@ mod tests { let all: HashSet = by_session.values().flatten().cloned().collect(); let rows = rows_with_ids(&ds, &all).await.unwrap(); - let (clean, held) = drop_secret_rows(rows).unwrap(); - assert_eq!(held.rows, by_session["dirty"].len(), "every secret split is held"); - assert!(held.summary.contains("PrivateKey"), "summary: {}", held.summary); + let gated = gate(&rows).unwrap(); + // The key is the block's whole text, so its redaction re-splits into fewer pieces than the + // rows it would replace: held back whole. + assert_eq!(gated.held.rows, by_session["dirty"].len(), "every secret split is held"); + assert!( + gated.held.summary.contains("PrivateKey"), + "summary: {}", + gated.held.summary + ); + let clean = gated_rows(rows, &gated, dataset::model::ENGLISH, |_| { + unreachable!("nothing redacted") + }) + .unwrap(); let clean_only: HashSet = by_session["clean"].iter().cloned().collect(); assert_eq!(dataset::ids_in(&clean), clean_only, "unrelated clean rows survive"); let rows = rows_with_ids(&ds, &clean_only).await.unwrap(); - assert_eq!(drop_secret_rows(rows).unwrap().1.rows, 0, "clean rows are not held"); + assert_eq!(gate(&rows).unwrap().held.rows, 0, "clean rows are not held"); } #[tokio::test] diff --git a/src/commands/scrub.rs b/src/commands/scrub.rs index 0ed216e2..68cf3467 100644 --- a/src/commands/scrub.rs +++ b/src/commands/scrub.rs @@ -67,7 +67,11 @@ pub async fn run() -> Result<()> { // window the next is found in. So a redaction counts only if the result scans clean. let clean = r.fully_redacted && scan::scan_blocks(&[r.text.as_str()], &scanner)?[0].is_empty(); if clean { - replacements.extend(chunk::resplit(&chunks[idxs[0]], &r.text)); + let tool_name = chunks[idxs[0]] + .tool_name + .as_deref() + .map(|n| scan::excise(n, findings).text); + replacements.extend(chunk::resplit(&chunks[idxs[0]], &r.text, tool_name)); redacted_detectors.extend(r.removed_detectors); redacted_blocks += 1; } else { diff --git a/src/inference/blas.rs b/src/inference/blas.rs index 2156a3d5..44e991b5 100644 --- a/src/inference/blas.rs +++ b/src/inference/blas.rs @@ -3,8 +3,8 @@ //! wires it to Accelerate (cblas_sgemm on AMX + vForce vvexpf), Linux to faer's pure-Rust GEMM //! plus a runtime-dispatched polynomial exp. This module holds the seam, the kernels, the model //! loaders and the length-grouped batching. The encoders built on them are cross-platform source: -//! BERT (the embedders and bge-reranker-base) in [`bert`], ModernBERT -//! (gte-reranker-modernbert-base) in [`modernbert`]. Gated behind the `blas` feature. +//! BERT (the embedders) in [`bert`], ModernBERT (gte-reranker-modernbert-base) in [`modernbert`]. +//! Gated behind the `blas` feature. use std::collections::HashMap; use std::io::{Read, Seek, SeekFrom}; @@ -17,7 +17,7 @@ use tokenizers::{Encoding, PaddingParams, Tokenizer, TruncationParams}; pub(super) mod bert; pub(super) mod modernbert; -pub use bert::{BlasEmbedder, BlasReranker}; +pub use bert::BlasEmbedder; pub use modernbert::GteReranker; // --------------------------------------------------------------------------------------------- diff --git a/src/inference/blas/bert.rs b/src/inference/blas/bert.rs index 7be4a43e..b4405a0a 100644 --- a/src/inference/blas/bert.rs +++ b/src/inference/blas/bert.rs @@ -1,6 +1,6 @@ -//! The BERT encoder on the CPU, for the embedders and bge-reranker-base: post-norm blocks with -//! biases, and learned positions added to the embeddings. It shares the seam, the kernels, the -//! loaders and the length-grouped batching of [`super`]. +//! The BERT encoder on the CPU, for the embedders: post-norm blocks with biases, and learned +//! positions added to the embeddings. It shares the seam, the kernels, the loaders and the +//! length-grouped batching of [`super`]. use std::collections::HashMap; @@ -9,22 +9,18 @@ use tokenizers::Tokenizer; use super::{ by_length, ceil_div, gelu, hf_snapshot, l2_normalize, layernorm, linear, load_tokenizer, load_weights, mean_pool, - par_add, seam, seq_workers, softmax_rows, NO_T, RERANK_GROUP, T, + par_add, seam, seq_workers, softmax_rows, NO_T, T, }; -use crate::inference::{Embedder, EmbeddingModel, Reranker}; +use crate::inference::{Embedder, EmbeddingModel}; -const PAD: i64 = 1; - -/// A BERT-family encoder config. Reranker is XLM-R (pad-offset positions); embedder is BERT. +/// The BERT encoder's shape. #[derive(Clone, Copy)] struct Cfg { - prefix: &'static str, h: usize, heads: usize, hd: usize, ffn: usize, layers: usize, - bert_pos: bool, /// Sequences per forward. A forward pads every sequence to the longest it holds, so /// [`by_length`] runs inputs length-sorted, this many at a time. Measured on real chunk /// lengths: fewer shrinks the GEMMs, more brings the padding back. `Scratch` is sized to one @@ -32,24 +28,12 @@ struct Cfg { group: usize, } -const RERANK: Cfg = Cfg { - prefix: "roberta.", - h: 768, - heads: 12, - hd: 64, - ffn: 3072, - layers: 12, - bert_pos: false, - group: RERANK_GROUP, -}; const EMBED: Cfg = Cfg { - prefix: "", h: 384, heads: 12, hd: 32, ffn: 1536, layers: 12, - bert_pos: true, group: 16, }; @@ -73,9 +57,9 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u let g = |k: &str| w.get(k).unwrap_or_else(|| panic!("missing weight {k}")).as_slice(); let (h, ffn, hd, heads) = (c.h, c.ffn, c.hd, c.heads); let m = n * l; - let word = g(&format!("{}embeddings.word_embeddings.weight", c.prefix)); - let pos = g(&format!("{}embeddings.position_embeddings.weight", c.prefix)); - let typ = g(&format!("{}embeddings.token_type_embeddings.weight", c.prefix)); + let word = g("embeddings.word_embeddings.weight"); + let pos = g("embeddings.position_embeddings.weight"); + let typ = g("embeddings.token_type_embeddings.weight"); for (buf, len) in [ (&mut s.hid, m * h), (&mut s.q, m * h), @@ -94,20 +78,11 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u scope.spawn(move || { for (si, seq) in grp.chunks_mut(l * h).enumerate() { let sq = gi * seqs_per + si; - let mut run = 0i64; for i in 0..l { let id = ids[sq * l + i]; - let posid = if c.bert_pos { - i - } else if id != PAD { - run += 1; - (PAD + run) as usize - } else { - PAD as usize - }; let dst = &mut seq[i * h..i * h + h]; let we = &word[id as usize * h..id as usize * h + h]; - let pe = &pos[posid * h..posid * h + h]; + let pe = &pos[i * h..i * h + h]; for cc in 0..h { dst[cc] = we[cc] + pe[cc] + typ[cc]; } @@ -119,13 +94,13 @@ fn encode(w: &HashMap>, c: Cfg, ids: &[i64], mask: &[i64], n: u layernorm( &mut s.hid, h, - g(&format!("{}embeddings.LayerNorm.weight", c.prefix)), - g(&format!("{}embeddings.LayerNorm.bias", c.prefix)), + g("embeddings.LayerNorm.weight"), + g("embeddings.LayerNorm.bias"), ); let scale = 1.0 / (hd as f32).sqrt(); for ly in 0..c.layers { - let p = format!("{}encoder.layer.{ly}", c.prefix); + let p = format!("encoder.layer.{ly}"); linear( &s.hid, m, @@ -289,64 +264,3 @@ impl Embedder for BlasEmbedder { }) } } - -/// bge-reranker-base cross-encoder: XLM-R encoder → CLS → out_proj(tanh(dense(cls))). -pub struct BlasReranker { - w: HashMap>, - tok: Tokenizer, - scratch: Scratch, -} - -impl BlasReranker { - pub fn new() -> Result { - let dir = hf_snapshot("BAAI/bge-reranker-base")?; - Ok(Self { - w: load_weights(&dir)?, - tok: load_tokenizer(&dir)?, - scratch: Scratch::default(), - }) - } -} - -impl Reranker for BlasReranker { - fn rerank(&mut self, query: &str, docs: &[&str]) -> Result> { - let pairs: Vec<(&str, &str)> = docs.iter().map(|d| (query, *d)).collect(); - let encs = self - .tok - .encode_batch(pairs, true) - .map_err(|e| anyhow!("tokenize: {e}"))?; - let (w, scratch) = (&self.w, &mut self.scratch); - let h = RERANK.h; - by_length(encs, RERANK.group, |ids, mask, n, l| { - encode(w, RERANK, ids, mask, n, l, scratch); - let mut cls = vec![0f32; n * h]; - for s in 0..n { - cls[s * h..s * h + h].copy_from_slice(&scratch.hid[(s * l) * h..(s * l) * h + h]); - } - let mut d = vec![0f32; n * h]; - linear( - &cls, - n, - h, - w["classifier.dense.weight"].as_slice(), - w["classifier.dense.bias"].as_slice(), - h, - &mut d, - ); - for v in d.iter_mut() { - *v = v.tanh(); - } - let mut scores = vec![0f32; n]; - linear( - &d, - n, - h, - w["classifier.out_proj.weight"].as_slice(), - w["classifier.out_proj.bias"].as_slice(), - 1, - &mut scores, - ); - scores - }) - } -} diff --git a/src/main.rs b/src/main.rs index 447e8a68..fd8123f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -211,9 +211,9 @@ enum Cmd { #[arg(value_name = "MEMORY")] memory: Option, }, - /// Redact secrets from your local memory in place — for rows indexed before redaction existed (or - /// flagged by an updated ruleset); needs no source transcript. Cleans the local memory only: it - /// does NOT scrub an already-published remote, which the push gate can only stop adding to. + /// Redact secrets from your local memory in place, such as the blocks a push holds back. Needs no + /// source transcript, and cleans the local memory only: it does NOT scrub an already-published + /// remote, which the push gate can only stop adding to. Scrub, /// Update funes in place: download the latest release binary for this platform and replace the /// running executable, then bring the installed agent integrations to their newest release. diff --git a/tests/scan_batch.rs b/tests/index_runs_no_scanner.rs similarity index 79% rename from tests/scan_batch.rs rename to tests/index_runs_no_scanner.rs index c7feb272..6b4c868d 100644 --- a/tests/scan_batch.rs +++ b/tests/index_runs_no_scanner.rs @@ -1,7 +1,6 @@ -//! A directory of many small turns files is scanned for secrets a batch at a time: the scanner is -//! spawned once per batch, not once per file, by `index --check` and by the index itself, and not -//! at all by a rerun that owes nothing. Own test binary: it sets `$FUNES_HOME` and -//! `$FUNES_TRUFFLEHOG`. +//! Indexing runs no secret scanner, neither `index --check` nor the index itself, over a directory +//! of more turns files than a batch holds: push is where secrets are redacted. A rerun that owes +//! nothing adds no rows. Own test binary: it sets `$FUNES_HOME` and `$FUNES_TRUFFLEHOG`. mod support; @@ -19,13 +18,13 @@ fn write_unit(path: &Path, line: &str, now: SystemTime, age: u64) { } #[tokio::test] -async fn a_directory_is_scanned_a_batch_at_a_time() { +async fn indexing_a_directory_runs_no_secret_scan() { let home = tempfile::tempdir().unwrap(); let bin = tempfile::tempdir().unwrap(); let source = tempfile::tempdir().unwrap(); std::env::set_var("FUNES_HOME", home.path()); - // A scanner that finds nothing and counts its spawns in a file beside itself. + // A scanner that counts its spawns in a file beside itself, so a scan by the index would show. let scanner = bin.path().join("trufflehog"); std::fs::write(&scanner, "#!/bin/sh\necho spawn >> \"$0.spawns\"\n").unwrap(); std::fs::set_permissions(&scanner, std::fs::Permissions::from_mode(0o755)).unwrap(); @@ -82,22 +81,21 @@ async fn a_directory_is_scanned_a_batch_at_a_time() { "a rejected unit is reported where it sits:\n{}", report.text ); - assert_eq!( - spawned(), - 2, - "41 units scan in two batches, the rejected one costing none" - ); + assert_eq!(spawned(), 0, "--check runs no secret scan"); - // The index scans the same two batches; a rerun that owes nothing scans nothing. + // The index runs none either, and a rerun that owes nothing adds no rows. std::fs::remove_file(&bad).unwrap(); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - assert_eq!(spawned(), 4); + assert_eq!(spawned(), 0, "the index runs no secret scan"); assert_eq!(support::stored_rows().await, units as usize); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - assert_eq!(spawned(), 4, "an unchanged rerun spawns no scanner"); - assert_eq!(support::stored_rows().await, units as usize, "and adds no rows"); + assert_eq!( + support::stored_rows().await, + units as usize, + "an unchanged rerun adds no rows" + ); } diff --git a/tests/redact_on_index.rs b/tests/redact_on_index.rs deleted file mode 100644 index 50536983..00000000 --- a/tests/redact_on_index.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Gated end-to-end: a secret planted in a transcript is redacted out of the memory at index time, -//! so it never reaches recall — or, via push, the Hub. Skipped unless trufflehog (the scanner) and -//! ssh-keygen (to mint a throwaway key) are both available. - -use std::io::Write; -use std::process::Command; - -#[tokio::test] -async fn planted_key_is_redacted_at_index_time() { - if funes::scan::Trufflehog::find().is_err() { - eprintln!("skip: trufflehog not found"); - return; - } - let home = tempfile::tempdir().unwrap(); - let source = tempfile::tempdir().unwrap(); - std::env::set_var("FUNES_HOME", home.path()); - - // Mint a throwaway private key (never committed) and plant it in a transcript. - let keyfile = home.path().join("throwaway_ed25519"); - let made = Command::new("ssh-keygen") - .args(["-t", "ed25519", "-N", "", "-q", "-f"]) - .arg(&keyfile) - .status() - .map(|s| s.success()) - .unwrap_or(false); - if !made { - eprintln!("skip: ssh-keygen unavailable"); - return; - } - let key = std::fs::read_to_string(&keyfile).unwrap(); - std::fs::remove_file(&keyfile).unwrap(); - // A distinctive slice of the key body — must not survive into the memory. - let key_body = key.lines().nth(1).unwrap().to_string(); - assert!(key_body.len() > 20); - - let session = "redact-session-0001"; - - let content = format!("here is my deploy key, keep it safe:\n{key}"); - let line = serde_json::json!({ - "format": 1, - "session_id": session, - "cwd": "/home/u/dev/demo", - "turn_uuid": "t1", - "seq": 0, - "ts": "2026-01-01T00:00:00Z", - "role": "user", - "blocks": [{"block_type": "text", "text": content}], - "harness": "claude", - }) - .to_string(); - let mut f = std::fs::File::create(source.path().join(format!("{session}.funes.jsonl"))).unwrap(); - writeln!(f, "{line}").unwrap(); - - // Index for real — redaction runs over the block text before chunking/embedding/storing. - funes::commands::index::run_index(source.path(), false, None) - .await - .unwrap(); - - // Read the stored turn back: the marker is present, the key body is gone. - let got = funes::commands::recall::get( - funes::memory::Memory::local(), - session.into(), - funes::commands::recall::TurnRange::default(), - ) - .await - .unwrap(); - assert!( - got.contains("[REDACTED:PrivateKey]"), - "expected a redaction marker, got: {got}" - ); - assert!(!got.contains(&key_body), "key body leaked into the memory: {got}"); -} diff --git a/tests/scrub.rs b/tests/scrub.rs index 0ccf2d82..2aeadcc9 100644 --- a/tests/scrub.rs +++ b/tests/scrub.rs @@ -2,9 +2,8 @@ //! its source, and re-embeds what it rewrote with the memory's own model (multilingual-e5-small //! here). Skipped unless trufflehog and ssh-keygen are available. //! -//! A dirty memory is created by indexing with `FUNES_TRUFFLEHOG=/usr/bin/true` — `true` ignores its -//! args and exits 0, so the scan finds nothing and the secret is stored unredacted (simulating a -//! memory built before redaction existed). Then scrub runs with the real scanner and cleans it. +//! Indexing stores the secret as written, so the memory is dirty until scrub runs with the real +//! scanner and cleans it. mod support; @@ -50,8 +49,6 @@ async fn scrub_redacts_an_existing_secret_in_place_with_the_memorys_model() { let session = "scrub-session-0001"; support::write_session(source.path(), session, &[format!("部署密钥:\n{key}")]); - // Index with a no-op scanner so the secret lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); @@ -68,8 +65,7 @@ async fn scrub_redacts_an_existing_secret_in_place_with_the_memorys_model() { "setup: the key should be on disk before scrub" ); - // Scrub with the real scanner: it must redact the key in place. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Scrub must redact the key in place. funes::commands::scrub::run().await.unwrap(); let clean = recall::get(Memory::local(), session.into(), TurnRange::default()) .await diff --git a/tests/scrub_drops_all.rs b/tests/scrub_drops_all.rs index 397c96d7..05a12f69 100644 --- a/tests/scrub_drops_all.rs +++ b/tests/scrub_drops_all.rs @@ -61,14 +61,12 @@ async fn scrub_dropping_every_block_leaves_a_valid_empty_memory() { let mut f = std::fs::File::create(source.path().join(format!("{session}.funes.jsonl"))).unwrap(); writeln!(f, "{line}").unwrap(); - // Index with a no-op scanner so the escaped key lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); + // Indexing stores the escaped key as written. funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); - // Scrub with the real scanner: every block is unredactable, so the memory is rewritten empty. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Every block is unredactable, so scrub rewrites the memory empty. funes::commands::scrub::run().await.unwrap(); // The empty Overwrite must leave a valid, reopenable memory with zero rows. diff --git a/tests/scrub_redacts_escaped_key.rs b/tests/scrub_redacts_escaped_key.rs index 551e55ca..1acaa217 100644 --- a/tests/scrub_redacts_escaped_key.rs +++ b/tests/scrub_redacts_escaped_key.rs @@ -66,8 +66,7 @@ async fn scrub_redacts_an_escaped_key_in_place() { writeln!(f, "{line}").unwrap(); } - // Index with a no-op scanner so the escaped key lands in the memory unredacted. - std::env::set_var("FUNES_TRUFFLEHOG", "/usr/bin/true"); + // Indexing stores the escaped key as written. funes::commands::index::run_index(source.path(), false, None) .await .unwrap(); @@ -86,8 +85,7 @@ async fn scrub_redacts_an_escaped_key_in_place() { "setup: the escaped key should be in the memory before scrub" ); - // Scrub with the real scanner: it matches the key's JSON-escaped form and excises it in place. - std::env::remove_var("FUNES_TRUFFLEHOG"); + // Scrub matches the key's JSON-escaped form and excises it in place. funes::commands::scrub::run().await.unwrap(); let after_key = funes::commands::recall::get( diff --git a/tests/turns_file_round_trip.rs b/tests/turns_file_round_trip.rs index 90fe6f87..5e6caee4 100644 --- a/tests/turns_file_round_trip.rs +++ b/tests/turns_file_round_trip.rs @@ -193,3 +193,29 @@ async fn a_row_closes_on_the_highest_seq_not_the_latest_stamp() { "the span closes on the highest seq's stamp:\n{listing}" ); } + +/// A block keeps the pieces it was first stored with, whatever text a re-read brings. funes 1.12 +/// and older stored a block with its secrets cut out, so reading its session again can split the +/// block into more pieces than the memory holds, the extra ones carrying the secret's raw tail. +#[tokio::test] +async fn a_stored_block_takes_no_pieces_from_a_longer_reread() { + let _one_at_a_time = HOME.lock().await; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("key.funes.jsonl"); + let mut turns = turns_of("funes_jsonl/valid.funes.jsonl"); + turns.truncate(1); + turns[0].blocks[0].text = "deploy key: [REDACTED:PrivateKey]".into(); + write_turns(&path, &turns); + let (home, first) = index_fresh(dir.path()).await; + + // The block as written, three pieces long, read again because its file changed. + turns[0].blocks[0].text = format!("deploy key: {}", "the raw tail of the secret ".repeat(100)); + write_turns(&path, &turns); + std::env::set_var("FUNES_HOME", home.path()); + run_index(dir.path(), false, None).await.unwrap(); + assert_eq!( + rows().await, + first, + "the stored block takes no piece of the longer text" + ); +}