From 219f4b3e6b3f4bce2b46966f3f7bca3ffb299ab5 Mon Sep 17 00:00:00 2001 From: Vlad Voropaev <43608848+voropaevv@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:01:10 +0400 Subject: [PATCH 1/3] fix(recall): stabilize equal-score search ranks --- src/commands/recall.rs | 73 ++++++++++++++++++++++++++++++++++++------ src/memory/dataset.rs | 10 ++++++ 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/src/commands/recall.rs b/src/commands/recall.rs index bd41289f..142f4e36 100644 --- a/src/commands/recall.rs +++ b/src/commands/recall.rs @@ -15,7 +15,7 @@ use crate::traces; use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, NaiveDate, Utc}; use lance::Dataset; -use std::cmp::Reverse; +use std::cmp::{Ordering, Reverse}; use std::collections::hash_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Write as _; @@ -952,29 +952,37 @@ async fn hybrid_candidates( /// Top-`limit` rows by vector distance, each with its row id (the fusion key). async fn vector_candidates(ds: &Dataset, qv: &[f32], limit: usize, filter: Option<&str>) -> Result> { - hits_of(&dataset::nearest(ds, qv, limit, filter, &hit_cols(ds)).await?) + let batches = dataset::nearest(ds, qv, limit, filter, &hit_cols(ds, "_distance")).await?; + hits_of(&batches, "_distance", f32::total_cmp) } /// Top-`limit` rows by BM25 score, each with its row id. Errors if the memory has no FTS index. async fn fts_candidates(ds: &Dataset, query: &str, limit: usize, filter: Option<&str>) -> Result> { - hits_of(&dataset::full_text(ds, query, limit, filter, &hit_cols(ds)).await?) + let batches = dataset::full_text(ds, query, limit, filter, &hit_cols(ds, "_score")).await?; + hits_of(&batches, "_score", |a, b| b.total_cmp(a)) } -/// `HIT_COLS`, minus `harness` on an un-migrated memory, which [`hits_of`] then reads as "". -fn hit_cols(ds: &Dataset) -> Vec<&'static str> { +/// Project the search score alongside the hit columns. Old memories can omit `harness`. +fn hit_cols(ds: &Dataset, score_column: &'static str) -> Vec<&'static str> { let has_harness = dataset::has_column(ds, "harness"); HIT_COLS .iter() .copied() .filter(|&c| c != "harness" || has_harness) + .chain([score_column]) .collect() } -/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank). -fn hits_of(batches: &[RecordBatch]) -> Result> { +/// Order each search's candidates before fusion so tied scores get stable ranks. +fn hits_of( + batches: &[RecordBatch], + score_column: &str, + compare_scores: impl Fn(&f32, &f32) -> Ordering, +) -> Result> { let mut out = Vec::new(); for batch in batches { let row_ids = dataset::row_ids(batch)?; + let scores = dataset::search_scores(batch, score_column)?; let (chunk_id, text, sess, proj, turn, ts, bt, harness) = ( dataset::strs(batch, "id"), dataset::strs(batch, "text"), @@ -986,8 +994,9 @@ fn hits_of(batches: &[RecordBatch]) -> Result> { dataset::strs(batch, "harness"), ); let seq = dataset::ints(batch, "seq"); - for i in 0..batch.num_rows() { + for (i, &score) in scores.iter().enumerate() { out.push(( + score, row_ids.get(i), Hit { id: chunk_id.get(i).to_string(), @@ -1006,7 +1015,10 @@ fn hits_of(batches: &[RecordBatch]) -> Result> { )); } } - Ok(out) + out.sort_unstable_by(|(a_score, a_id, _), (b_score, b_id, _)| { + compare_scores(a_score, b_score).then(a_id.cmp(b_id)) + }); + Ok(out.into_iter().map(|(_, id, hit)| (id, hit)).collect()) } /// Reciprocal-rank fusion (k=60): each list contributes `1/(rank + 60)` to a row's score; return @@ -2198,6 +2210,49 @@ mod tests { assert_eq!(ids(rrf_fuse(v, f, 5)), ["0", "1", "2", "3", "4"]); } + #[test] + fn search_hits_break_score_ties_before_fusion() { + use arrow_array::{ArrayRef, Float32Array, StringArray, UInt64Array}; + use std::sync::Arc; + + let batch = |score_column, rows: &[(u64, f32)]| { + RecordBatch::try_from_iter([ + ( + "_rowid", + Arc::new(UInt64Array::from_iter_values(rows.iter().map(|(id, _)| *id))) as ArrayRef, + ), + ( + "id", + Arc::new(StringArray::from_iter_values(rows.iter().map(|(id, _)| id.to_string()))) as ArrayRef, + ), + ( + score_column, + Arc::new(Float32Array::from_iter_values(rows.iter().map(|(_, score)| *score))) as ArrayRef, + ), + ]) + .unwrap() + }; + + for (score_column, best, worse) in [("_distance", 0.0, 0.5), ("_score", 0.5, 0.0)] { + // Lance may deliver the same score tie in either order and across batches. + for (a, b) in [(11, 3), (3, 11)] { + let batches = [ + batch(score_column, &[(a, best)]), + batch(score_column, &[(b, best), (1, worse)]), + ]; + let compare: fn(&f32, &f32) -> Ordering = if score_column == "_distance" { + f32::total_cmp + } else { + |a, b| b.total_cmp(a) + }; + let hits = hits_of(&batches, score_column, compare).unwrap(); + let row_ids: Vec<_> = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!(row_ids, [3, 11, 1], "{score_column}"); + assert_eq!(rrf_fuse(hits, vec![], 1)[0].id, "3"); + } + } + } + #[test] fn share_by_type_takes_each_types_next_row_in_turn() { // (block type, fused score), best first. diff --git a/src/memory/dataset.rs b/src/memory/dataset.rs index cfa5d29a..fad266e1 100644 --- a/src/memory/dataset.rs +++ b/src/memory/dataset.rs @@ -367,6 +367,16 @@ impl Vectors<'_> { } } +/// The score of each returned search row, in batch order. +pub(crate) fn search_scores<'a>(batch: &'a RecordBatch, name: &str) -> Result<&'a [f32]> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .filter(|c| c.null_count() == 0) + .map(|c| c.values().as_ref()) + .with_context(|| format!("a search result without valid {name} scores")) +} + /// The row ids of a batch scanned with them. pub struct RowIds<'a>(&'a UInt64Array); From af03096970fd90000c605bc5488cd7408139ff63 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Thu, 12 Nov 2026 09:43:51 +0000 Subject: [PATCH 2/3] review(recall): reorder only the rows of equal score Lance returns each search best first, so only the order within a run of equal scores is left to settle. Sorting those runs by row id needs no score direction, so the comparator each search passed goes away. The test picked its comparator by column name, apart from the one each search passed, so a flipped comparator in fts_candidates would still have passed it. Co-Authored-By: Claude Opus 5.5 --- src/commands/recall.rs | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/commands/recall.rs b/src/commands/recall.rs index 142f4e36..0c99db9a 100644 --- a/src/commands/recall.rs +++ b/src/commands/recall.rs @@ -15,7 +15,7 @@ use crate::traces; use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, NaiveDate, Utc}; use lance::Dataset; -use std::cmp::{Ordering, Reverse}; +use std::cmp::Reverse; use std::collections::hash_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Write as _; @@ -953,13 +953,13 @@ async fn hybrid_candidates( /// Top-`limit` rows by vector distance, each with its row id (the fusion key). async fn vector_candidates(ds: &Dataset, qv: &[f32], limit: usize, filter: Option<&str>) -> Result> { let batches = dataset::nearest(ds, qv, limit, filter, &hit_cols(ds, "_distance")).await?; - hits_of(&batches, "_distance", f32::total_cmp) + hits_of(&batches, "_distance") } /// Top-`limit` rows by BM25 score, each with its row id. Errors if the memory has no FTS index. async fn fts_candidates(ds: &Dataset, query: &str, limit: usize, filter: Option<&str>) -> Result> { let batches = dataset::full_text(ds, query, limit, filter, &hit_cols(ds, "_score")).await?; - hits_of(&batches, "_score", |a, b| b.total_cmp(a)) + hits_of(&batches, "_score") } /// Project the search score alongside the hit columns. Old memories can omit `harness`. @@ -973,12 +973,9 @@ fn hit_cols(ds: &Dataset, score_column: &'static str) -> Vec<&'static str> { .collect() } -/// Order each search's candidates before fusion so tied scores get stable ranks. -fn hits_of( - batches: &[RecordBatch], - score_column: &str, - compare_scores: impl Fn(&f32, &f32) -> Ordering, -) -> Result> { +/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank), equal scores +/// by row id. +fn hits_of(batches: &[RecordBatch], score_column: &str) -> Result> { let mut out = Vec::new(); for batch in batches { let row_ids = dataset::row_ids(batch)?; @@ -1015,9 +1012,11 @@ fn hits_of( )); } } - out.sort_unstable_by(|(a_score, a_id, _), (b_score, b_id, _)| { - compare_scores(a_score, b_score).then(a_id.cmp(b_id)) - }); + // Equal BM25 scores come back in an order that depends on which of Lance's index segments + // finishes first. + for tie in out.chunk_by_mut(|(a, ..), (b, ..)| a == b) { + tie.sort_unstable_by_key(|(_, id, _)| *id); + } Ok(out.into_iter().map(|(_, id, hit)| (id, hit)).collect()) } @@ -2240,12 +2239,7 @@ mod tests { batch(score_column, &[(a, best)]), batch(score_column, &[(b, best), (1, worse)]), ]; - let compare: fn(&f32, &f32) -> Ordering = if score_column == "_distance" { - f32::total_cmp - } else { - |a, b| b.total_cmp(a) - }; - let hits = hits_of(&batches, score_column, compare).unwrap(); + let hits = hits_of(&batches, score_column).unwrap(); let row_ids: Vec<_> = hits.iter().map(|(id, _)| *id).collect(); assert_eq!(row_ids, [3, 11, 1], "{score_column}"); assert_eq!(rrf_fuse(hits, vec![], 1)[0].id, "3"); From 4a5bbab4a96578ee4c9c3142fb382caf9038cad7 Mon Sep 17 00:00:00 2001 From: David Corvoysier Date: Thu, 12 Nov 2026 10:05:49 +0000 Subject: [PATCH 3/3] review: leave the search score column to dataset nearest and full_text project their own score column, and search_scores reads it back from either one. recall named Lance's _distance and _score columns, which belong with the Lance mechanics, like _rowid behind row_ids. So hit_cols and the two searches in recall are back to what main has. The projection is still needed. Lance 13 adds the score column on its own today, but warns that it will stop. Co-Authored-By: Claude Opus 5.5 --- src/commands/recall.rs | 17 +++++++---------- src/memory/dataset.rs | 34 +++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/commands/recall.rs b/src/commands/recall.rs index 0c99db9a..894310a0 100644 --- a/src/commands/recall.rs +++ b/src/commands/recall.rs @@ -952,34 +952,31 @@ async fn hybrid_candidates( /// Top-`limit` rows by vector distance, each with its row id (the fusion key). async fn vector_candidates(ds: &Dataset, qv: &[f32], limit: usize, filter: Option<&str>) -> Result> { - let batches = dataset::nearest(ds, qv, limit, filter, &hit_cols(ds, "_distance")).await?; - hits_of(&batches, "_distance") + hits_of(&dataset::nearest(ds, qv, limit, filter, &hit_cols(ds)).await?) } /// Top-`limit` rows by BM25 score, each with its row id. Errors if the memory has no FTS index. async fn fts_candidates(ds: &Dataset, query: &str, limit: usize, filter: Option<&str>) -> Result> { - let batches = dataset::full_text(ds, query, limit, filter, &hit_cols(ds, "_score")).await?; - hits_of(&batches, "_score") + hits_of(&dataset::full_text(ds, query, limit, filter, &hit_cols(ds)).await?) } -/// Project the search score alongside the hit columns. Old memories can omit `harness`. -fn hit_cols(ds: &Dataset, score_column: &'static str) -> Vec<&'static str> { +/// `HIT_COLS`, minus `harness` on an un-migrated memory, which [`hits_of`] then reads as "". +fn hit_cols(ds: &Dataset) -> Vec<&'static str> { let has_harness = dataset::has_column(ds, "harness"); HIT_COLS .iter() .copied() .filter(|&c| c != "harness" || has_harness) - .chain([score_column]) .collect() } /// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank), equal scores /// by row id. -fn hits_of(batches: &[RecordBatch], score_column: &str) -> Result> { +fn hits_of(batches: &[RecordBatch]) -> Result> { let mut out = Vec::new(); for batch in batches { let row_ids = dataset::row_ids(batch)?; - let scores = dataset::search_scores(batch, score_column)?; + let scores = dataset::search_scores(batch)?; let (chunk_id, text, sess, proj, turn, ts, bt, harness) = ( dataset::strs(batch, "id"), dataset::strs(batch, "text"), @@ -2239,7 +2236,7 @@ mod tests { batch(score_column, &[(a, best)]), batch(score_column, &[(b, best), (1, worse)]), ]; - let hits = hits_of(&batches, score_column).unwrap(); + let hits = hits_of(&batches).unwrap(); let row_ids: Vec<_> = hits.iter().map(|(id, _)| *id).collect(); assert_eq!(row_ids, [3, 11, 1], "{score_column}"); assert_eq!(rrf_fuse(hits, vec![], 1)[0].id, "3"); diff --git a/src/memory/dataset.rs b/src/memory/dataset.rs index fad266e1..2b7aedad 100644 --- a/src/memory/dataset.rs +++ b/src/memory/dataset.rs @@ -27,9 +27,11 @@ use lance::dataset::{ use lance::index::vector::VectorIndexParams; use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; use lance_index::optimize::OptimizeOptions; +use lance_index::scalar::inverted::SCORE_COL; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; +use lance_index::vector::DIST_COL; use lance_index::{IndexParams, IndexType}; use lance_io::object_store::{ObjectStoreParams, WrappingObjectStore}; use lance_linalg::distance::MetricType; @@ -134,9 +136,10 @@ pub async fn scan_rows_with_ids(ds: &Dataset, columns: &[&str], filter: Option<& /// rank the shortlist: the codes alone misorder close neighbours. const REFINE_FACTOR: u32 = 10; -/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id. -/// `filter` is applied before the search, not to its result: a selective filter would otherwise -/// drop most of a globally-nearest pool and return far fewer than `limit` rows when matches exist. +/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id and +/// its distance ([`search_scores`]). `filter` is applied before the search, not to its result: a +/// selective filter would otherwise drop most of a globally-nearest pool and return far fewer than +/// `limit` rows when matches exist. pub async fn nearest( ds: &Dataset, query: &[f32], @@ -152,13 +155,14 @@ pub async fn nearest( scan.prefilter(true); scan.filter(f)?; } - scan.project(columns)?; + scan.project(&[columns, &[DIST_COL]].concat())?; scan.with_row_id(); collect(scan).await } -/// The `limit` rows best matching `query` by BM25, best first, each with its row id. `filter` is -/// applied before the search, as in [`nearest`]. Errors on a memory with no text index. +/// The `limit` rows best matching `query` by BM25, best first, each with its row id and its score +/// ([`search_scores`]). `filter` is applied before the search, as in [`nearest`]. Errors on a memory +/// with no text index. pub async fn full_text( ds: &Dataset, query: &str, @@ -172,7 +176,7 @@ pub async fn full_text( scan.prefilter(true); scan.filter(f)?; } - scan.project(columns)?; + scan.project(&[columns, &[SCORE_COL]].concat())?; scan.with_row_id(); scan.limit(Some(limit as i64), None)?; collect(scan).await @@ -367,14 +371,16 @@ impl Vectors<'_> { } } -/// The score of each returned search row, in batch order. -pub(crate) fn search_scores<'a>(batch: &'a RecordBatch, name: &str) -> Result<&'a [f32]> { - batch - .column_by_name(name) +/// The score of each row of a batch from [`nearest`] (its distance) or [`full_text`] (its BM25 +/// score), in batch order. +pub(crate) fn search_scores(batch: &RecordBatch) -> Result<&[f32]> { + [DIST_COL, SCORE_COL] + .into_iter() + .find_map(|name| batch.column_by_name(name)) .and_then(|c| c.as_any().downcast_ref::()) .filter(|c| c.null_count() == 0) .map(|c| c.values().as_ref()) - .with_context(|| format!("a search result without valid {name} scores")) + .context("a search result without scores") } /// The row ids of a batch scanned with them. @@ -1186,9 +1192,7 @@ mod tests { build_indexes(&mut ds, |_| {}).await.unwrap(); assert_eq!(sub_index_counts(&ds).await.unwrap()[VECTOR_INDEX], 1); - let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid", "_distance"]) - .await - .unwrap(); + let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid"]).await.unwrap(); let distance = batches[0] .column_by_name("_distance") .and_then(|c| c.as_any().downcast_ref::())