diff --git a/src/commands/recall.rs b/src/commands/recall.rs index bd41289f..894310a0 100644 --- a/src/commands/recall.rs +++ b/src/commands/recall.rs @@ -970,11 +970,13 @@ fn hit_cols(ds: &Dataset) -> Vec<&'static str> { .collect() } -/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank). +/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank), equal scores +/// by row id. fn hits_of(batches: &[RecordBatch]) -> Result> { let mut out = Vec::new(); for batch in batches { let row_ids = dataset::row_ids(batch)?; + let scores = dataset::search_scores(batch)?; let (chunk_id, text, sess, proj, turn, ts, bt, harness) = ( dataset::strs(batch, "id"), dataset::strs(batch, "text"), @@ -986,8 +988,9 @@ fn hits_of(batches: &[RecordBatch]) -> Result> { dataset::strs(batch, "harness"), ); let seq = dataset::ints(batch, "seq"); - for i in 0..batch.num_rows() { + for (i, &score) in scores.iter().enumerate() { out.push(( + score, row_ids.get(i), Hit { id: chunk_id.get(i).to_string(), @@ -1006,7 +1009,12 @@ fn hits_of(batches: &[RecordBatch]) -> Result> { )); } } - Ok(out) + // Equal BM25 scores come back in an order that depends on which of Lance's index segments + // finishes first. + for tie in out.chunk_by_mut(|(a, ..), (b, ..)| a == b) { + tie.sort_unstable_by_key(|(_, id, _)| *id); + } + Ok(out.into_iter().map(|(_, id, hit)| (id, hit)).collect()) } /// Reciprocal-rank fusion (k=60): each list contributes `1/(rank + 60)` to a row's score; return @@ -2198,6 +2206,44 @@ mod tests { assert_eq!(ids(rrf_fuse(v, f, 5)), ["0", "1", "2", "3", "4"]); } + #[test] + fn search_hits_break_score_ties_before_fusion() { + use arrow_array::{ArrayRef, Float32Array, StringArray, UInt64Array}; + use std::sync::Arc; + + let batch = |score_column, rows: &[(u64, f32)]| { + RecordBatch::try_from_iter([ + ( + "_rowid", + Arc::new(UInt64Array::from_iter_values(rows.iter().map(|(id, _)| *id))) as ArrayRef, + ), + ( + "id", + Arc::new(StringArray::from_iter_values(rows.iter().map(|(id, _)| id.to_string()))) as ArrayRef, + ), + ( + score_column, + Arc::new(Float32Array::from_iter_values(rows.iter().map(|(_, score)| *score))) as ArrayRef, + ), + ]) + .unwrap() + }; + + for (score_column, best, worse) in [("_distance", 0.0, 0.5), ("_score", 0.5, 0.0)] { + // Lance may deliver the same score tie in either order and across batches. + for (a, b) in [(11, 3), (3, 11)] { + let batches = [ + batch(score_column, &[(a, best)]), + batch(score_column, &[(b, best), (1, worse)]), + ]; + let hits = hits_of(&batches).unwrap(); + let row_ids: Vec<_> = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!(row_ids, [3, 11, 1], "{score_column}"); + assert_eq!(rrf_fuse(hits, vec![], 1)[0].id, "3"); + } + } + } + #[test] fn share_by_type_takes_each_types_next_row_in_turn() { // (block type, fused score), best first. diff --git a/src/memory/dataset.rs b/src/memory/dataset.rs index cfa5d29a..2b7aedad 100644 --- a/src/memory/dataset.rs +++ b/src/memory/dataset.rs @@ -27,9 +27,11 @@ use lance::dataset::{ use lance::index::vector::VectorIndexParams; use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; use lance_index::optimize::OptimizeOptions; +use lance_index::scalar::inverted::SCORE_COL; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; +use lance_index::vector::DIST_COL; use lance_index::{IndexParams, IndexType}; use lance_io::object_store::{ObjectStoreParams, WrappingObjectStore}; use lance_linalg::distance::MetricType; @@ -134,9 +136,10 @@ pub async fn scan_rows_with_ids(ds: &Dataset, columns: &[&str], filter: Option<& /// rank the shortlist: the codes alone misorder close neighbours. const REFINE_FACTOR: u32 = 10; -/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id. -/// `filter` is applied before the search, not to its result: a selective filter would otherwise -/// drop most of a globally-nearest pool and return far fewer than `limit` rows when matches exist. +/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id and +/// its distance ([`search_scores`]). `filter` is applied before the search, not to its result: a +/// selective filter would otherwise drop most of a globally-nearest pool and return far fewer than +/// `limit` rows when matches exist. pub async fn nearest( ds: &Dataset, query: &[f32], @@ -152,13 +155,14 @@ pub async fn nearest( scan.prefilter(true); scan.filter(f)?; } - scan.project(columns)?; + scan.project(&[columns, &[DIST_COL]].concat())?; scan.with_row_id(); collect(scan).await } -/// The `limit` rows best matching `query` by BM25, best first, each with its row id. `filter` is -/// applied before the search, as in [`nearest`]. Errors on a memory with no text index. +/// The `limit` rows best matching `query` by BM25, best first, each with its row id and its score +/// ([`search_scores`]). `filter` is applied before the search, as in [`nearest`]. Errors on a memory +/// with no text index. pub async fn full_text( ds: &Dataset, query: &str, @@ -172,7 +176,7 @@ pub async fn full_text( scan.prefilter(true); scan.filter(f)?; } - scan.project(columns)?; + scan.project(&[columns, &[SCORE_COL]].concat())?; scan.with_row_id(); scan.limit(Some(limit as i64), None)?; collect(scan).await @@ -367,6 +371,18 @@ impl Vectors<'_> { } } +/// The score of each row of a batch from [`nearest`] (its distance) or [`full_text`] (its BM25 +/// score), in batch order. +pub(crate) fn search_scores(batch: &RecordBatch) -> Result<&[f32]> { + [DIST_COL, SCORE_COL] + .into_iter() + .find_map(|name| batch.column_by_name(name)) + .and_then(|c| c.as_any().downcast_ref::()) + .filter(|c| c.null_count() == 0) + .map(|c| c.values().as_ref()) + .context("a search result without scores") +} + /// The row ids of a batch scanned with them. pub struct RowIds<'a>(&'a UInt64Array); @@ -1176,9 +1192,7 @@ mod tests { build_indexes(&mut ds, |_| {}).await.unwrap(); assert_eq!(sub_index_counts(&ds).await.unwrap()[VECTOR_INDEX], 1); - let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid", "_distance"]) - .await - .unwrap(); + let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid"]).await.unwrap(); let distance = batches[0] .column_by_name("_distance") .and_then(|c| c.as_any().downcast_ref::())