From 5b4c0c6ecd23d564756fc3523840e70ab82e9998 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:49:41 +0900 Subject: [PATCH 1/2] fix(core): search every file when a multi-grep pattern has no bigrams Co-Authored-By: Claude Opus 5 --- crates/fff-core/src/grep/grep_tests.rs | 75 +++++++++++++++++++++++++ crates/fff-core/src/index/candidates.rs | 23 ++++---- 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/crates/fff-core/src/grep/grep_tests.rs b/crates/fff-core/src/grep/grep_tests.rs index a7c42da34..9eeb21598 100644 --- a/crates/fff-core/src/grep/grep_tests.rs +++ b/crates/fff-core/src/grep/grep_tests.rs @@ -518,3 +518,78 @@ fn regex_fallback_keeps_file_path_scope_issue_756() { "regex fallback must not leak outside the FilePath scope" ); } + +/// Candidate bitsets are OR-ed across the patterns of a multi-pattern grep. A +/// pattern the bigram index cannot prefilter (no printable-ASCII bigram, e.g. a +/// CJK needle) matches any file, so leaving it out of the union hides every file +/// that only it matches. +#[test] +fn multi_grep_keeps_files_for_a_pattern_without_bigrams() { + let dir = tempfile::tempdir().unwrap(); + let base = crate::path_utils::canonicalize(dir.path()).unwrap(); + + // Only a/b/c carry the ASCII needle, so its bigrams stay below the + // ubiquity threshold that `compress` drops. + let base_contents: &[(&str, &str)] = &[ + ("a.txt", "hello unicorn world"), + ("b.txt", "another unicorn line"), + ("c.txt", "one more unicorn here"), + ("d.txt", "nothing special in here"), + ("e.txt", "日本語のテキスト"), + ("f.txt", "rainbow sky above"), + ]; + for (name, content) in base_contents { + let mut f = std::fs::File::create(base.join(name)).unwrap(); + writeln!(f, "{}", content).unwrap(); + } + + let mut picker = FilePicker::new(FilePickerOptions { + base_path: base.to_str().unwrap().into(), + watch: false, + ..Default::default() + }) + .unwrap(); + picker.collect_files().unwrap(); + assert_eq!(picker.get_files().len(), base_contents.len()); + + let consec_builder = BigramIndexBuilder::new(base_contents.len()); + let skip_builder = BigramIndexBuilder::new(base_contents.len()); + for (i, (_, content)) in base_contents.iter().enumerate() { + consec_builder.add_file_content(&skip_builder, i, content.as_bytes()); + } + let mut index = consec_builder.compress(Some(0)); + index.set_skip_index(skip_builder.compress(Some(0))); + picker.set_bigram_index(index); + + let options = crate::GrepSearchOptions { + mode: super::GrepMode::PlainText, + smart_case: true, + page_limit: 100, + ..Default::default() + }; + + let matched_paths = |patterns: &[&str]| -> Vec { + let result = picker.multi_grep(patterns, &[], &options); + let mut paths: Vec = result + .files + .iter() + .map(|f| f.relative_path(&picker)) + .collect(); + paths.sort(); + paths + }; + + assert_eq!( + matched_paths(&["unicorn", "日本語"]), + vec!["a.txt", "b.txt", "c.txt", "e.txt"], + "the CJK needle has no usable bigram, so its file must still be searched" + ); + + // Pins that the prefilter still narrows (both hold before and after): + // every pattern indexable => files matching none of them stay out. + assert_eq!( + matched_paths(&["unicorn", "rainbow"]), + vec!["a.txt", "b.txt", "c.txt", "f.txt"] + ); + assert_eq!(matched_paths(&["unicorn"]), vec!["a.txt", "b.txt", "c.txt"]); +} diff --git a/crates/fff-core/src/index/candidates.rs b/crates/fff-core/src/index/candidates.rs index e6a2669bd..1542d51fd 100644 --- a/crates/fff-core/src/index/candidates.rs +++ b/crates/fff-core/src/index/candidates.rs @@ -38,17 +38,18 @@ pub(crate) fn literal_candidates( let mut combined: Option> = None; for pattern in patterns { - if let Some(candidates) = index.query(pattern.as_bytes()) { - combined = Some(match combined { - None => candidates, - Some(mut acc) => { - acc.iter_mut() - .zip(candidates.iter()) - .for_each(|(a, b)| *a |= *b); - acc - } - }); - } + // A pattern the index can't narrow matches any file, and the union with + // it is every file — so bail out of prefiltering instead of dropping it. + let candidates = index.query(pattern.as_bytes())?; + combined = Some(match combined { + None => candidates, + Some(mut acc) => { + acc.iter_mut() + .zip(candidates.iter()) + .for_each(|(a, b)| *a |= *b); + acc + } + }); } let mut candidates = combined?; From b9413a933cc60cb9db0697eedf1782d25706cacd Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:34:59 +0900 Subject: [PATCH 2/2] test(core): assert the multi-grep prefilter through the files it searches The regression test compresses its index the way build_bigram_index does and checks total_files_searched, so it shows the CJK needle disables the prefilter and an all-indexable query still narrows. The private test no longer carries a doc comment. Co-Authored-By: Claude Fable 5.1 --- crates/fff-core/src/grep/grep_tests.rs | 42 +++++++++++----------- crates/fff-core/src/index/bigram_filter.rs | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/fff-core/src/grep/grep_tests.rs b/crates/fff-core/src/grep/grep_tests.rs index 9eeb21598..3ea6f0e2b 100644 --- a/crates/fff-core/src/grep/grep_tests.rs +++ b/crates/fff-core/src/grep/grep_tests.rs @@ -519,10 +519,6 @@ fn regex_fallback_keeps_file_path_scope_issue_756() { ); } -/// Candidate bitsets are OR-ed across the patterns of a multi-pattern grep. A -/// pattern the bigram index cannot prefilter (no printable-ASCII bigram, e.g. a -/// CJK needle) matches any file, so leaving it out of the union hides every file -/// that only it matches. #[test] fn multi_grep_keeps_files_for_a_pattern_without_bigrams() { let dir = tempfile::tempdir().unwrap(); @@ -557,8 +553,11 @@ fn multi_grep_keeps_files_for_a_pattern_without_bigrams() { for (i, (_, content)) in base_contents.iter().enumerate() { consec_builder.add_file_content(&skip_builder, i, content.as_bytes()); } - let mut index = consec_builder.compress(Some(0)); - index.set_skip_index(skip_builder.compress(Some(0))); + // Compressed as build_bigram_index does, so this is the production prefilter. + let mut index = consec_builder.compress(None); + index.set_skip_index(skip_builder.compress(Some( + crate::index::bigram_filter::SKIP_INDEX_MIN_DENSITY_PCT, + ))); picker.set_bigram_index(index); let options = crate::GrepSearchOptions { @@ -568,7 +567,7 @@ fn multi_grep_keeps_files_for_a_pattern_without_bigrams() { ..Default::default() }; - let matched_paths = |patterns: &[&str]| -> Vec { + let grep = |patterns: &[&str]| -> (Vec, usize) { let result = picker.multi_grep(patterns, &[], &options); let mut paths: Vec = result .files @@ -576,20 +575,21 @@ fn multi_grep_keeps_files_for_a_pattern_without_bigrams() { .map(|f| f.relative_path(&picker)) .collect(); paths.sort(); - paths + (paths, result.total_files_searched) }; - assert_eq!( - matched_paths(&["unicorn", "日本語"]), - vec!["a.txt", "b.txt", "c.txt", "e.txt"], - "the CJK needle has no usable bigram, so its file must still be searched" - ); - - // Pins that the prefilter still narrows (both hold before and after): - // every pattern indexable => files matching none of them stay out. - assert_eq!( - matched_paths(&["unicorn", "rainbow"]), - vec!["a.txt", "b.txt", "c.txt", "f.txt"] - ); - assert_eq!(matched_paths(&["unicorn"]), vec!["a.txt", "b.txt", "c.txt"]); + // The CJK needle has no printable-ASCII bigram, so nothing can be + // prefiltered: every file is searched and e.txt is found. + let (paths, searched) = grep(&["unicorn", "日本語"]); + assert_eq!(paths, vec!["a.txt", "b.txt", "c.txt", "e.txt"]); + assert_eq!(searched, base_contents.len()); + + // Every pattern indexable: only the files carrying some pattern's + // bigrams are searched, so the prefilter still narrows. + let (paths, searched) = grep(&["unicorn", "rainbow"]); + assert_eq!(paths, vec!["a.txt", "b.txt", "c.txt", "f.txt"]); + assert_eq!(searched, 4); + let (paths, searched) = grep(&["unicorn"]); + assert_eq!(paths, vec!["a.txt", "b.txt", "c.txt"]); + assert_eq!(searched, 3); } diff --git a/crates/fff-core/src/index/bigram_filter.rs b/crates/fff-core/src/index/bigram_filter.rs index 2c409151b..e85078011 100644 --- a/crates/fff-core/src/index/bigram_filter.rs +++ b/crates/fff-core/src/index/bigram_filter.rs @@ -794,7 +794,7 @@ const BIGRAM_CHUNK_FILES: usize = 4 * 64; /// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add /// little filtering power but ~25-30% of index memory, so we drop /// anything appearing in < 12 % of populated files. -const SKIP_INDEX_MIN_DENSITY_PCT: u32 = 12; +pub(crate) const SKIP_INDEX_MIN_DENSITY_PCT: u32 = 12; thread_local! { /// Reusable read buffer that is allocated per thread and used for reading files