Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 49 additions & 3 deletions src/commands/recall.rs
Original file line number Diff line number Diff line change
Expand Up @@ -970,11 +970,13 @@ fn hit_cols(ds: &Dataset) -> Vec<&'static str> {
.collect()
}

/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank).
/// The `(row id, Hit)` of every row of `batches`, in their order (the search's rank), equal scores
/// by row id.
fn hits_of(batches: &[RecordBatch]) -> Result<Vec<(u64, Hit)>> {
let mut out = Vec::new();
for batch in batches {
let row_ids = dataset::row_ids(batch)?;
let scores = dataset::search_scores(batch)?;
let (chunk_id, text, sess, proj, turn, ts, bt, harness) = (
dataset::strs(batch, "id"),
dataset::strs(batch, "text"),
Expand All @@ -986,8 +988,9 @@ fn hits_of(batches: &[RecordBatch]) -> Result<Vec<(u64, Hit)>> {
dataset::strs(batch, "harness"),
);
let seq = dataset::ints(batch, "seq");
for i in 0..batch.num_rows() {
for (i, &score) in scores.iter().enumerate() {
out.push((
score,
row_ids.get(i),
Hit {
id: chunk_id.get(i).to_string(),
Expand All @@ -1006,7 +1009,12 @@ fn hits_of(batches: &[RecordBatch]) -> Result<Vec<(u64, Hit)>> {
));
}
}
Ok(out)
// Equal BM25 scores come back in an order that depends on which of Lance's index segments
// finishes first.
for tie in out.chunk_by_mut(|(a, ..), (b, ..)| a == b) {
tie.sort_unstable_by_key(|(_, id, _)| *id);
}
Ok(out.into_iter().map(|(_, id, hit)| (id, hit)).collect())
}

/// Reciprocal-rank fusion (k=60): each list contributes `1/(rank + 60)` to a row's score; return
Expand Down Expand Up @@ -2198,6 +2206,44 @@ mod tests {
assert_eq!(ids(rrf_fuse(v, f, 5)), ["0", "1", "2", "3", "4"]);
}

#[test]
fn search_hits_break_score_ties_before_fusion() {
use arrow_array::{ArrayRef, Float32Array, StringArray, UInt64Array};
use std::sync::Arc;

let batch = |score_column, rows: &[(u64, f32)]| {
RecordBatch::try_from_iter([
(
"_rowid",
Arc::new(UInt64Array::from_iter_values(rows.iter().map(|(id, _)| *id))) as ArrayRef,
),
(
"id",
Arc::new(StringArray::from_iter_values(rows.iter().map(|(id, _)| id.to_string()))) as ArrayRef,
),
(
score_column,
Arc::new(Float32Array::from_iter_values(rows.iter().map(|(_, score)| *score))) as ArrayRef,
),
])
.unwrap()
};

for (score_column, best, worse) in [("_distance", 0.0, 0.5), ("_score", 0.5, 0.0)] {
// Lance may deliver the same score tie in either order and across batches.
for (a, b) in [(11, 3), (3, 11)] {
let batches = [
batch(score_column, &[(a, best)]),
batch(score_column, &[(b, best), (1, worse)]),
];
let hits = hits_of(&batches).unwrap();
let row_ids: Vec<_> = hits.iter().map(|(id, _)| *id).collect();
assert_eq!(row_ids, [3, 11, 1], "{score_column}");
assert_eq!(rrf_fuse(hits, vec![], 1)[0].id, "3");
}
}
}

#[test]
fn share_by_type_takes_each_types_next_row_in_turn() {
// (block type, fused score), best first.
Expand Down
34 changes: 24 additions & 10 deletions src/memory/dataset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ use lance::dataset::{
use lance::index::vector::VectorIndexParams;
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
use lance_index::optimize::OptimizeOptions;
use lance_index::scalar::inverted::SCORE_COL;
use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
use lance_index::vector::ivf::IvfBuildParams;
use lance_index::vector::pq::PQBuildParams;
use lance_index::vector::DIST_COL;
use lance_index::{IndexParams, IndexType};
use lance_io::object_store::{ObjectStoreParams, WrappingObjectStore};
use lance_linalg::distance::MetricType;
Expand Down Expand Up @@ -134,9 +136,10 @@ pub async fn scan_rows_with_ids(ds: &Dataset, columns: &[&str], filter: Option<&
/// rank the shortlist: the codes alone misorder close neighbours.
const REFINE_FACTOR: u32 = 10;

/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id.
/// `filter` is applied before the search, not to its result: a selective filter would otherwise
/// drop most of a globally-nearest pool and return far fewer than `limit` rows when matches exist.
/// The `limit` rows nearest to `query` by vector distance, nearest first, each with its row id and
/// its distance ([`search_scores`]). `filter` is applied before the search, not to its result: a
/// selective filter would otherwise drop most of a globally-nearest pool and return far fewer than
/// `limit` rows when matches exist.
pub async fn nearest(
ds: &Dataset,
query: &[f32],
Expand All @@ -152,13 +155,14 @@ pub async fn nearest(
scan.prefilter(true);
scan.filter(f)?;
}
scan.project(columns)?;
scan.project(&[columns, &[DIST_COL]].concat())?;
scan.with_row_id();
collect(scan).await
}

/// The `limit` rows best matching `query` by BM25, best first, each with its row id. `filter` is
/// applied before the search, as in [`nearest`]. Errors on a memory with no text index.
/// The `limit` rows best matching `query` by BM25, best first, each with its row id and its score
/// ([`search_scores`]). `filter` is applied before the search, as in [`nearest`]. Errors on a memory
/// with no text index.
pub async fn full_text(
ds: &Dataset,
query: &str,
Expand All @@ -172,7 +176,7 @@ pub async fn full_text(
scan.prefilter(true);
scan.filter(f)?;
}
scan.project(columns)?;
scan.project(&[columns, &[SCORE_COL]].concat())?;
scan.with_row_id();
scan.limit(Some(limit as i64), None)?;
collect(scan).await
Expand Down Expand Up @@ -367,6 +371,18 @@ impl Vectors<'_> {
}
}

/// The score of each row of a batch from [`nearest`] (its distance) or [`full_text`] (its BM25
/// score), in batch order.
pub(crate) fn search_scores(batch: &RecordBatch) -> Result<&[f32]> {
[DIST_COL, SCORE_COL]
.into_iter()
.find_map(|name| batch.column_by_name(name))
.and_then(|c| c.as_any().downcast_ref::<Float32Array>())
.filter(|c| c.null_count() == 0)
.map(|c| c.values().as_ref())
.context("a search result without scores")
}

/// The row ids of a batch scanned with them.
pub struct RowIds<'a>(&'a UInt64Array);

Expand Down Expand Up @@ -1176,9 +1192,7 @@ mod tests {
build_indexes(&mut ds, |_| {}).await.unwrap();
assert_eq!(sub_index_counts(&ds).await.unwrap()[VECTOR_INDEX], 1);

let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid", "_distance"])
.await
.unwrap();
let batches = nearest(&ds, &vectors(n)[7], 10, None, &["turn_uuid"]).await.unwrap();
let distance = batches[0]
.column_by_name("_distance")
.and_then(|c| c.as_any().downcast_ref::<Float32Array>())
Expand Down
Loading