From a3aedfcfe08b91ce82046028967b0b5b011fca74 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 10:58:22 -0500 Subject: [PATCH 1/9] feat(csr): store the csr inside each lance dataset with persist_csr Engines rebuild every CSR from a full scan of the relationship tables at start-up. With persist_csr=True, each full build of a Lance relationship dataset is written to /_lance_csr/.arrow, a zstd compressed Arrow IPC file, and later engines load it instead of building. Changes since the stored version apply as a delta under the usual 25% threshold, read from the file's recorded fragments, so this survives compaction and cleanup_old_versions(). Unusable files are rebuilt, and failed writes fall back to an ordinary build. delete_stored_csr() removes the stored files. csr_stats() reports update "loaded" for a CSR read from a file. CsrIndex now keeps its arrays as Arrow buffers, and Rust gains EngineOptions and CypherEngine::with_options. On GDC SF1 read lazily, engine start goes from 1.55 s to 0.58 s, the files take 150 MiB, and query times are unchanged. --- Cargo.lock | 2 + crates/lance-csr-python/Cargo.lock | 2 + crates/lance-csr-python/src/graph.rs | 22 +- crates/lance-csr/Cargo.toml | 4 +- crates/lance-csr/src/csr_expand.rs | 75 +++- crates/lance-csr/src/csr_index.rs | 43 +- crates/lance-csr/src/csr_store.rs | 339 ++++++++++++++++ crates/lance-csr/src/engine.rs | 384 ++++++++++++++++-- crates/lance-csr/src/lib.rs | 1 + docs/csr.md | 68 +++- docs/internals/indexes.md | 27 ++ python/benchmarks/csr_persist.py | 152 +++++++ python/python/lance_csr/_internal/graph.pyi | 4 +- python/python/lance_csr/_types.py | 2 +- python/python/lance_csr/cypher.py | 39 +- .../python/tests/test_engine_csr_persist.py | 153 +++++++ 16 files changed, 1247 insertions(+), 70 deletions(-) create mode 100644 crates/lance-csr/src/csr_store.rs create mode 100644 python/benchmarks/csr_persist.py create mode 100644 python/python/tests/test_engine_csr_persist.py diff --git a/Cargo.lock b/Cargo.lock index c84f2ee..245ea7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3966,7 +3966,9 @@ dependencies = [ "lance-linalg", "lance-namespace", "lance-table", + "log", "nom 7.1.3", + "object_store 0.14.2", "serde", "serde_json", "snafu 0.8.9", diff --git a/crates/lance-csr-python/Cargo.lock b/crates/lance-csr-python/Cargo.lock index 835fd7e..701192a 100644 --- a/crates/lance-csr-python/Cargo.lock +++ b/crates/lance-csr-python/Cargo.lock @@ -3850,7 +3850,9 @@ dependencies = [ "lance-linalg", "lance-namespace", "lance-table", + "log", "nom 7.1.3", + "object_store 0.14.2", "serde", "serde_json", "snafu 0.8.9", diff --git a/crates/lance-csr-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs index d8af19c..0725dd3 100644 --- a/crates/lance-csr-python/src/graph.rs +++ b/crates/lance-csr-python/src/graph.rs @@ -26,7 +26,7 @@ use datafusion::execution::context::SessionContext; use futures::StreamExt; use lance_csr::{ ast::{DistanceMetric as RustDistanceMetric, GraphPattern, ReadingClause}, - engine::{CypherEngine as RustCypherEngine, EngineSource, MAX_TARGET_PARTITIONS}, + engine::{CypherEngine as RustCypherEngine, EngineOptions, EngineSource, MAX_TARGET_PARTITIONS}, CypherQuery as RustCypherQuery, ExecutionStrategy as RustExecutionStrategy, GraphConfig as RustGraphConfig, GraphError as RustGraphError, SqlDialect as RustSqlDialect, SqlQuery as RustSqlQuery, VectorSearch as RustVectorSearch, @@ -1015,14 +1015,18 @@ pub struct CypherEngine { #[pymethods] impl CypherEngine { #[new] - #[pyo3(signature = (config, datasets, use_csr = false, *, target_partitions = None))] + #[pyo3(signature = (config, datasets, use_csr = false, *, target_partitions = None, persist_csr = false))] fn new( py: Python, config: &GraphConfig, datasets: &Bound<'_, PyDict>, use_csr: bool, target_partitions: Option<&Bound<'_, PyAny>>, + persist_csr: bool, ) -> PyResult { + if persist_csr && !use_csr { + return Err(PyValueError::new_err("persist_csr requires use_csr=True")); + } let target_partitions = target_partitions .map(|value| { if value.is_instance_of::() || !value.is_instance_of::() { @@ -1051,11 +1055,14 @@ impl CypherEngine { let inner = RT .block_on( Some(py), - RustCypherEngine::with_target_partitions( + RustCypherEngine::with_options( config.inner.clone(), sources, - use_csr, - target_partitions, + EngineOptions { + use_csr, + target_partitions, + persist_csr, + }, ), )? .map_err(graph_error_to_pyerr)?; @@ -1122,6 +1129,11 @@ impl CypherEngine { .map_err(graph_error_to_pyerr) } + fn delete_stored_csr(&self, py: Python) -> PyResult { + RT.block_on(Some(py), self.inner.delete_stored_csr())? + .map_err(graph_error_to_pyerr) + } + fn node_index_stats(&self, py: Python) -> PyResult> { let stats = PyDict::new(py); for (label, entry) in self.inner.node_index_stats() { diff --git a/crates/lance-csr/Cargo.toml b/crates/lance-csr/Cargo.toml index 731c945..c101dc8 100644 --- a/crates/lance-csr/Cargo.toml +++ b/crates/lance-csr/Cargo.toml @@ -13,7 +13,7 @@ categories = ["database", "data-structures", "science"] [dependencies] ahash = "0.8" -arrow = { version = "58", features = ["prettyprint"] } +arrow = { version = "58", features = ["prettyprint", "ipc_compression"] } arrow-array = "58" arrow-schema = "58" async-trait = "0.1" @@ -38,7 +38,9 @@ lance = "13.0.0" lance-linalg = "13.0.0" lance-namespace = "13.0.0" lance-table = "13.0.0" +log = "0.4" nom = "7.1" +object_store = "0.14" serde = { version = "1", features = ["derive"] } serde_json = "1" snafu = "0.8" diff --git a/crates/lance-csr/src/csr_expand.rs b/crates/lance-csr/src/csr_expand.rs index bddc22f..5f24dec 100644 --- a/crates/lance-csr/src/csr_expand.rs +++ b/crates/lance-csr/src/csr_expand.rs @@ -252,6 +252,35 @@ impl Adjacency { next } + /// An adjacency with no delta over a stored base: `keys` are the source keys in + /// CSR position order, as [`Self::base_parts`] returns them. `None` if a key + /// repeats. + pub(crate) fn from_base(keys: &[i64], csr: CsrIndex) -> Option { + let mut dense = FastMap::with_capacity_and_hasher(keys.len(), Default::default()); + dense.extend(keys.iter().enumerate().map(|(i, &k)| (k, i as u64))); + if dense.len() != keys.len() { + return None; + } + Some(Self { + base: Arc::new(CsrBase { dense, csr }), + added: FastMap::default(), + removed: FastMap::default(), + added_count: 0, + removed_count: 0, + removed_added_count: 0, + }) + } + + /// The base CSR: its source keys in CSR position order, and the index over them. + /// The delta is not included. + pub(crate) fn base_parts(&self) -> (Vec, &CsrIndex) { + let mut keys = vec![0; self.base.dense.len()]; + for (&key, &position) in &self.base.dense { + keys[position as usize] = key; + } + (keys, &self.base.csr) + } + /// Call `f` for each target endpoint value reachable from `key`, duplicates included. pub fn for_each_neighbor(&self, key: i64, f: impl FnMut(u64)) { self.neighbors_from(key, &mut 0).for_each(f); @@ -461,15 +490,42 @@ pub async fn edge_changes( src: &str, dst: &str, ) -> Result> { - let endpoint_ids = |ds: &Dataset| -> Option<[(i32, DataType); 2]> { - let field = |name: &str| ds.schema().field(name).map(|f| (f.id, f.data_type())); - Some([field(src)?, field(dst)?]) + let Some(old_fields) = endpoint_fields(old, src, dst) else { + return Ok(None); }; - let old_ids = endpoint_ids(old); - if old_ids.is_none() || old_ids != endpoint_ids(new) { + let old_frags: Vec = old + .get_fragments() + .into_iter() + .map(|f| f.metadata().clone()) + .collect(); + edge_changes_since(&old_frags, &old_fields, new, src, dst).await +} + +/// Lance field id and type of a relationship table's two endpoint columns. +pub(crate) type EndpointFields = [(i32, String); 2]; + +pub(crate) fn endpoint_fields(ds: &Dataset, src: &str, dst: &str) -> Option { + let field = |name: &str| { + ds.schema() + .field(name) + .map(|f| (f.id, f.data_type().to_string())) + }; + Some([field(src)?, field(dst)?]) +} + +/// [`edge_changes`] from a version known only by its fragments and endpoint fields, +/// as a stored CSR records them. +pub(crate) async fn edge_changes_since( + old_fragments: &[Fragment], + old_fields: &EndpointFields, + new: &Dataset, + src: &str, + dst: &str, +) -> Result> { + if endpoint_fields(new, src, dst).as_ref() != Some(old_fields) { return Ok(None); } - let fields: Vec = old_ids.unwrap().iter().map(|(id, _)| *id).collect(); + let fields: Vec = old_fields.iter().map(|(id, _)| *id).collect(); let touches = |f: &[i32]| f.iter().any(|id| fields.contains(id)); // The parts of a fragment that determine its endpoint values. let endpoint_data = |frag: &Fragment| { @@ -482,11 +538,8 @@ pub async fn edge_changes( format!("{:?}{:?}", files, overlays) }; - let old_frags: HashMap = old - .get_fragments() - .into_iter() - .map(|f| (f.id() as u64, f.metadata().clone())) - .collect(); + let old_frags: HashMap = + old_fragments.iter().map(|f| (f.id, f.clone())).collect(); let new_frags: HashMap = new .get_fragments() .into_iter() diff --git a/crates/lance-csr/src/csr_index.rs b/crates/lance-csr/src/csr_index.rs index 6c57f77..c30a05c 100644 --- a/crates/lance-csr/src/csr_index.rs +++ b/crates/lance-csr/src/csr_index.rs @@ -20,6 +20,8 @@ use std::collections::HashMap; +use arrow::buffer::ScalarBuffer; + /// In-memory CSR adjacency index for fast neighbor lookup. /// /// Stores graph topology in three arrays: @@ -27,14 +29,41 @@ use std::collections::HashMap; /// - `neighbors[offsets[v]..offsets[v+1]]` = destination vertex IDs /// - `edge_rows`, aligned with `neighbors` = the row (a Lance row address, or a /// row index for in-memory input) each edge came from +/// +/// The arrays are Arrow buffers, so an index read from a stored file shares the +/// file's memory instead of copying it. #[derive(Debug, Clone)] pub struct CsrIndex { - offsets: Vec, - neighbors: Vec, - edge_rows: Vec, + offsets: ScalarBuffer, + neighbors: ScalarBuffer, + edge_rows: ScalarBuffer, } impl CsrIndex { + /// An index from its three arrays, as [`Self::parts`] returns them, or `None` + /// if `offsets` is empty, descends or doesn't end at `neighbors.len()`, or + /// `edge_rows` and `neighbors` differ in length. + pub fn from_parts( + offsets: ScalarBuffer, + neighbors: ScalarBuffer, + edge_rows: ScalarBuffer, + ) -> Option { + let well_formed = offsets.first() == Some(&0) + && offsets.last() == Some(&(neighbors.len() as u64)) + && offsets.windows(2).all(|w| w[0] <= w[1]) + && edge_rows.len() == neighbors.len(); + well_formed.then_some(Self { + offsets, + neighbors, + edge_rows, + }) + } + + /// The offsets, neighbours and edge rows. + pub fn parts(&self) -> (&ScalarBuffer, &ScalarBuffer, &ScalarBuffer) { + (&self.offsets, &self.neighbors, &self.edge_rows) + } + fn range(&self, vertex_id: u64) -> std::ops::Range { let v = vertex_id as usize; if v >= self.offsets.len() - 1 { @@ -125,13 +154,13 @@ impl CsrIndexBuilder { } offsets[num_vertices as usize] = running; - let edge_rows = self.edges.iter().map(|&(_, _, row)| row).collect(); + let edge_rows: Vec = self.edges.iter().map(|&(_, _, row)| row).collect(); neighbors.extend(self.edges.iter().map(|&(_, dst, _)| dst)); CsrIndex { - offsets, - neighbors, - edge_rows, + offsets: offsets.into(), + neighbors: neighbors.into(), + edge_rows: edge_rows.into(), } } } diff --git a/crates/lance-csr/src/csr_store.rs b/crates/lance-csr/src/csr_store.rs new file mode 100644 index 0000000..347fddf --- /dev/null +++ b/crates/lance-csr/src/csr_store.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Persisted CSR files, for engines built with `persist_csr`. +//! +//! The base CSR of a relationship dataset is stored inside the dataset's own +//! directory, one Arrow IPC file per dataset version: `_lance_csr/.arrow`. +//! Lance's `cleanup_old_versions()` deletes only the files it recognises (data, +//! index, deletion and transaction files, temporary manifests), so it leaves this +//! folder alone. +//! +//! One record batch holds both directions, one row per edge in CSR position order: +//! `out_source`/`in_source` are run-end encoded source keys (the run ends are the +//! CSR offsets), next to `*_neighbor` and `*_edge_row`. The schema metadata records +//! the dataset version, the endpoint columns and fields, and the fragments the base +//! was built from, so [`edge_changes_since`] can bring an older file up to date. +//! +//! A file is used only if it parses, its arrays are well formed and its endpoint +//! fields match; anything else is ignored and the CSR is rebuilt. The arrays are +//! zstd-compressed (4.7 times smaller on SF1), so loading decompresses them straight +//! into the CSR's buffers. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, Int64Array, RecordBatch, RunArray, UInt64Array}; +use arrow::buffer::{Buffer, ScalarBuffer}; +use arrow::datatypes::{DataType, Field, Int64Type, Schema, UInt64Type}; +use arrow::ipc::convert::fb_to_schema; +use arrow::ipc::reader::{read_footer_length, FileDecoder}; +use arrow::ipc::root_as_footer; +use arrow::ipc::writer::{FileWriter, IpcWriteOptions}; +use arrow::ipc::CompressionType; +use lance::dataset::Dataset; +use lance_table::format::Fragment; +use object_store::path::Path; + +use crate::csr_expand::{endpoint_fields, Adjacency, EndpointFields}; +use crate::csr_index::CsrIndex; +use crate::error::{GraphError, Result}; + +/// Folder at the dataset root that holds the files. +const DIR: &str = "_lance_csr"; +/// Bumped whenever the layout changes; files of another format are ignored. +const FORMAT: &str = "1"; + +const KEY_FORMAT: &str = "lance_csr.format"; +const KEY_VERSION: &str = "lance_csr.version"; +const KEY_ENDPOINTS: &str = "lance_csr.endpoints"; +const KEY_FIELDS: &str = "lance_csr.fields"; +const KEY_FRAGMENTS: &str = "lance_csr.fragments"; + +/// A base CSR read from a file, with what is needed to bring it up to date. +pub(crate) struct StoredCsr { + /// Dataset version the base was built from. + pub version: u64, + pub fragments: Vec, + pub fields: EndpointFields, + pub outgoing: Adjacency, + pub incoming: Adjacency, +} + +fn store_error(message: String) -> GraphError { + GraphError::ExecutionError { + message, + location: snafu::Location::new(file!(), line!(), column!()), + } +} + +/// `/_lance_csr`. Lance exposes `/data` but not the base itself. +fn dir(ds: &Dataset) -> Path { + let data = ds.data_dir(); + let parts: Vec<_> = data.parts().collect(); + Path::from_iter(parts[..parts.len().saturating_sub(1)].iter().cloned()).join(DIR) +} + +fn file_path(ds: &Dataset, version: u64) -> Path { + dir(ds).join(format!("{version}.arrow")) +} + +/// Versions of the files in `_lance_csr`, unsorted. +async fn stored_versions(ds: &Dataset) -> Vec { + let Ok(store) = ds.object_store(None).await else { + return Vec::new(); + }; + // A missing folder lists as empty or fails, depending on the store. + store + .read_dir(dir(ds)) + .await + .unwrap_or_default() + .iter() + .filter_map(|name| name.strip_suffix(".arrow")?.parse().ok()) + .collect() +} + +/// The newest stored base at or before `ds`'s version for columns `src -> dst`, or +/// `None` when there is none or it can't be used. +pub(crate) async fn load(ds: &Dataset, src: &str, dst: &str) -> Option { + let current = ds.manifest().version; + let version = stored_versions(ds) + .await + .into_iter() + .filter(|&v| v <= current) + .max()?; + let store = ds.object_store(None).await.ok()?; + let bytes = store.read_one_all(&file_path(ds, version)).await.ok()?; + match decode(Buffer::from(bytes), src, dst) { + Ok(stored) if stored.version == version => Some(stored), + Ok(_) => None, + Err(e) => { + log::warn!("Ignoring stored CSR {version} of {}: {e}", ds.uri()); + None + } + } +} + +/// Write the base CSR of `ds` at its pinned version, then delete older files. +/// +/// `outgoing` and `incoming` must have no delta: they are the full build of `ds`. +pub(crate) async fn save( + ds: &Dataset, + src: &str, + dst: &str, + outgoing: &Adjacency, + incoming: &Adjacency, +) -> Result<()> { + let version = ds.manifest().version; + let fields = endpoint_fields(ds, src, dst) + .ok_or_else(|| store_error(format!("No endpoint columns {src}, {dst}")))?; + let fragments: Vec = ds + .get_fragments() + .into_iter() + .map(|f| f.metadata().clone()) + .collect(); + let bytes = encode(version, src, dst, &fields, &fragments, outgoing, incoming)?; + + let store = ds + .object_store(None) + .await + .map_err(|e| store_error(format!("Lance: {e}")))?; + let path = file_path(ds, version); + // shortcut: the compressed file is held whole in memory before the upload (150 MiB for SF1); write it through a multipart upload if files get much larger. + let payload = object_store::PutPayload::from(bytes); + let put = store.put_if_absent(&path, payload.clone()).await; + let put = match put { + // An earlier file for this version wasn't usable (we only save after a build), + // or another process just wrote the same content; replace it either way. + Err(object_store::Error::AlreadyExists { .. }) => { + store + .delete(&path) + .await + .map_err(|e| store_error(format!("Lance: {e}")))?; + store.put_if_absent(&path, payload).await + } + other => other, + }; + put.map_err(|e| store_error(format!("Failed to write {path}: {e}")))?; + + for old in stored_versions(ds) + .await + .into_iter() + .filter(|&v| v < version) + { + // Best effort: a file left behind is ignored once a newer one exists. + let _ = store.delete(&file_path(ds, old)).await; + } + Ok(()) +} + +/// Delete every stored CSR file of `ds`, and the folder where the store allows it. +/// Returns how many files were deleted. +pub(crate) async fn delete_all(ds: &Dataset) -> Result { + let store = ds + .object_store(None) + .await + .map_err(|e| store_error(format!("Lance: {e}")))?; + let mut deleted = 0; + for version in stored_versions(ds).await { + store + .delete(&file_path(ds, version)) + .await + .map_err(|e| store_error(format!("Failed to delete a stored CSR: {e}")))?; + deleted += 1; + } + // Object stores have no folders; on local disk this drops the now empty one. + let _ = store.remove_dir_all(dir(ds)).await; + Ok(deleted) +} + +fn source_field(name: &str) -> Field { + Field::new( + name, + DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int64, false)), + Arc::new(Field::new("values", DataType::Int64, true)), + ), + false, + ) +} + +fn encode( + version: u64, + src: &str, + dst: &str, + fields: &EndpointFields, + fragments: &[Fragment], + outgoing: &Adjacency, + incoming: &Adjacency, +) -> Result> { + let metadata = HashMap::from([ + (KEY_FORMAT.to_string(), FORMAT.to_string()), + (KEY_VERSION.to_string(), version.to_string()), + (KEY_ENDPOINTS.to_string(), to_json(&[src, dst])?), + (KEY_FIELDS.to_string(), to_json(fields)?), + (KEY_FRAGMENTS.to_string(), to_json(fragments)?), + ]); + let mut schema_fields = Vec::new(); + let mut columns = Vec::new(); + for (prefix, adjacency) in [("out", outgoing), ("in", incoming)] { + schema_fields.extend([ + source_field(&format!("{prefix}_source")), + Field::new(format!("{prefix}_neighbor"), DataType::UInt64, false), + Field::new(format!("{prefix}_edge_row"), DataType::UInt64, false), + ]); + columns.extend(encode_direction(adjacency)?); + } + let schema = Arc::new(Schema::new_with_metadata(schema_fields, metadata)); + let batch = RecordBatch::try_new(schema.clone(), columns) + .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + let options = IpcWriteOptions::default() + .try_with_compression(Some(CompressionType::ZSTD)) + .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + let mut bytes = Vec::new(); + let mut writer = FileWriter::try_new_with_options(&mut bytes, &schema, options) + .and_then(|mut w| w.write(&batch).map(|_| w)) + .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + writer + .finish() + .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + drop(writer); + Ok(bytes) +} + +/// Source keys, neighbours and edge rows of one direction, one row per edge. +fn encode_direction(adjacency: &Adjacency) -> Result<[ArrayRef; 3]> { + let (keys, csr) = adjacency.base_parts(); + let (offsets, neighbors, edge_rows) = csr.parts(); + // Every base source has at least one edge, so `offsets[1..]` are strictly + // ascending: valid run ends, shared with the offsets rather than copied. + let ends = ScalarBuffer::::new(offsets.inner().clone(), 1, keys.len()); + let sources = + RunArray::::try_new(&Int64Array::new(ends, None), &Int64Array::from(keys)) + .map_err(|e| store_error(format!("Failed to encode CSR sources: {e}")))?; + Ok([ + Arc::new(sources), + Arc::new(UInt64Array::new(neighbors.clone(), None)), + Arc::new(UInt64Array::new(edge_rows.clone(), None)), + ]) +} + +fn decode(bytes: Buffer, src: &str, dst: &str) -> Result { + let bad = |what: &str| store_error(format!("malformed CSR file: {what}")); + let ipc = |e: arrow::error::ArrowError| store_error(format!("malformed CSR file: {e}")); + if bytes.len() < 10 { + return Err(bad("too short")); + } + let trailer = bytes.len() - 10; + let footer_len = + read_footer_length(bytes[trailer..].try_into().expect("10 bytes")).map_err(ipc)?; + let footer = root_as_footer( + &bytes[trailer + .checked_sub(footer_len) + .ok_or_else(|| bad("footer"))?..trailer], + ) + .map_err(|e| store_error(format!("malformed CSR file: {e}")))?; + let schema = Arc::new(fb_to_schema( + footer.schema().ok_or_else(|| bad("no schema"))?, + )); + let meta = |key: &str| schema.metadata().get(key).ok_or_else(|| bad(key)); + if meta(KEY_FORMAT)? != FORMAT { + return Err(bad("unknown format")); + } + let endpoints: [String; 2] = from_json(meta(KEY_ENDPOINTS)?)?; + if endpoints != [src, dst] { + return Err(bad("built for other endpoint columns")); + } + let version = meta(KEY_VERSION)?.parse().map_err(|_| bad(KEY_VERSION))?; + let fields = from_json(meta(KEY_FIELDS)?)?; + let fragments = from_json(meta(KEY_FRAGMENTS)?)?; + + let blocks = footer.recordBatches().ok_or_else(|| bad("no batches"))?; + if blocks.len() != 1 { + return Err(bad("expected one batch")); + } + let block = blocks.get(0); + let length = block.bodyLength() as usize + block.metaDataLength() as usize; + if block.offset() as usize + length > bytes.len() { + return Err(bad("batch out of range")); + } + let data = bytes.slice_with_length(block.offset() as usize, length); + let batch = FileDecoder::new(schema, footer.version()) + .read_record_batch(block, &data) + .map_err(ipc)? + .ok_or_else(|| bad("empty batch"))?; + Ok(StoredCsr { + version, + fragments, + fields, + outgoing: decode_direction(&batch, "out").ok_or_else(|| bad("outgoing arrays"))?, + incoming: decode_direction(&batch, "in").ok_or_else(|| bad("incoming arrays"))?, + }) +} + +fn decode_direction(batch: &RecordBatch, prefix: &str) -> Option { + let column = |name: &str| batch.column_by_name(&format!("{prefix}_{name}")); + let sources = column("source")?.as_run_opt::()?; + let ends = sources.run_ends(); + let keys = sources.values().as_primitive_opt::()?; + if sources.offset() != 0 || keys.null_count() != 0 || keys.len() != ends.values().len() { + return None; + } + let mut offsets = Vec::with_capacity(keys.len() + 1); + offsets.push(0); + offsets.extend(ends.values().iter().map(|&end| end as u64)); + let u64s = |name: &str| { + let array = column(name)?.as_primitive_opt::()?; + (array.null_count() == 0).then(|| array.values().clone()) + }; + let csr = CsrIndex::from_parts(offsets.into(), u64s("neighbor")?, u64s("edge_row")?)?; + Adjacency::from_base(keys.values(), csr) +} + +fn from_json(text: &str) -> Result { + serde_json::from_str(text).map_err(|e| store_error(format!("malformed CSR file: {e}"))) +} + +fn to_json(value: &T) -> Result { + serde_json::to_string(value).map_err(|e| store_error(format!("Failed to encode CSR: {e}"))) +} diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs index 1eedae9..913db2f 100644 --- a/crates/lance-csr/src/engine.rs +++ b/crates/lance-csr/src/engine.rs @@ -47,9 +47,10 @@ use serde_json::Value as JsonValue; use crate::config::GraphConfig; use crate::csr_expand::{ - edge_changes, is_supported_id_type, Adjacency, AdjacencyCatalog, CsrQueryPlanner, EdgeRows, - NodeRows, PruneCsrTargetColumns, PushCsrTargetFilters, + edge_changes, edge_changes_since, is_supported_id_type, Adjacency, AdjacencyCatalog, + CsrQueryPlanner, EdgeChanges, EdgeRows, NodeRows, PruneCsrTargetColumns, PushCsrTargetFilters, }; +use crate::csr_store; use crate::error::{GraphError, Result}; use crate::key_statistics::KeyStatisticsProvider; use crate::lance_vector_search::VectorSearch; @@ -172,7 +173,8 @@ pub struct CsrStats { /// since removed. Lance's `num_unindexed_rows`, counted in edges. pub unindexed_edges: u64, /// How the last refresh or `optimize_csr()` produced it: `build` (first), `rebuild` - /// (full), `delta` (changes applied) or `reused` (no edge change). + /// (full), `loaded` (from a stored file, with any changes since applied as a + /// delta), `delta` (changes applied) or `reused` (no edge change). pub update: &'static str, /// Relationship dataset version it reflects (`None` for Arrow input). pub version: Option, @@ -273,7 +275,7 @@ struct CsrEntry { base_version: Option, /// Wall time of the last build or delta application. build_ms: f64, - /// How the last refresh or optimize produced this entry: build, rebuild, delta or reused. + /// How the last refresh or optimize produced this entry: build, rebuild, loaded, delta or reused. update: &'static str, } @@ -326,16 +328,33 @@ async fn try_apply_csr_delta( ..old.clone() }); } + let (outgoing, incoming) = apply_csr_changes(&old.outgoing, &old.incoming, &changes)?; + Some(CsrEntry { + source: source.clone(), + outgoing: Arc::new(outgoing), + incoming: Arc::new(incoming), + edges: source.edge_rows(), + base_version: old.base_version, + build_ms: elapsed(), + update: "delta", + }) +} + +/// `changes` applied to both directions, or `None` when the accumulated delta would +/// pass `CSR_REBUILD_FRACTION` of the base. +fn apply_csr_changes( + outgoing: &Adjacency, + incoming: &Adjacency, + changes: &EdgeChanges, +) -> Option<(Adjacency, Adjacency)> { let too_large = - |pending: u64| pending as f64 > CSR_REBUILD_FRACTION * old.outgoing.base_edges() as f64; - if too_large( - old.outgoing.delta_edges() + old.outgoing.tombstones() + changes.added.len() as u64, - ) { + |pending: u64| pending as f64 > CSR_REBUILD_FRACTION * outgoing.base_edges() as f64; + if too_large(outgoing.delta_edges() + outgoing.tombstones() + changes.added.len() as u64) { return None; } // Removed rows are only counted by applying them; one direction is enough to decide. - let outgoing = old.outgoing.with_changes(&changes.added, &changes.removed); - if too_large(outgoing.delta_edges() + outgoing.tombstones()) { + let new_outgoing = outgoing.with_changes(&changes.added, &changes.removed); + if too_large(new_outgoing.delta_edges() + new_outgoing.tombstones()) { return None; } let flipped: Vec<_> = changes @@ -343,15 +362,36 @@ async fn try_apply_csr_delta( .iter() .map(|&(a, b, row)| (b, a, row)) .collect(); - let incoming = old.incoming.with_changes(&flipped, &changes.removed); + let new_incoming = incoming.with_changes(&flipped, &changes.removed); + Some((new_outgoing, new_incoming)) +} + +/// The CSR stored for `source`, brought up to date with the changes since. +/// +/// `None` when nothing usable is stored, the changes can't be computed, or they would +/// pass the rebuild threshold. +async fn load_stored_csr(source: &EngineSource, src: &str, dst: &str) -> Option { + let EngineSource::Lance { dataset, .. } = source else { + return None; + }; + let started = std::time::Instant::now(); + let stored = csr_store::load(dataset, src, dst).await?; + let changes = edge_changes_since(&stored.fragments, &stored.fields, dataset, src, dst) + .await + .ok()??; + let (outgoing, incoming) = if changes.is_empty() { + (stored.outgoing, stored.incoming) + } else { + apply_csr_changes(&stored.outgoing, &stored.incoming, &changes)? + }; Some(CsrEntry { source: source.clone(), outgoing: Arc::new(outgoing), incoming: Arc::new(incoming), edges: source.edge_rows(), - base_version: old.base_version, - build_ms: elapsed(), - update: "delta", + base_version: Some(stored.version), + build_ms: started.elapsed().as_secs_f64() * 1e3, + update: "loaded", }) } @@ -464,13 +504,12 @@ fn validate_sources(config: &GraphConfig, schemas: &HashMap) async fn build_engine_state( config: &GraphConfig, sources: HashMap, - use_csr: bool, - target_partitions: Option, + options: &EngineOptions, previous: Option<&EngineState>, fold_csr_deltas: bool, ) -> Result { let mut session_config = SessionConfig::new(); - if let Some(partitions) = target_partitions { + if let Some(partitions) = options.target_partitions { session_config = session_config.with_target_partitions(partitions); } // The CSR planner only changes plans that contain CSR nodes, so install it always. @@ -508,9 +547,17 @@ async fn build_engine_state( } validate_sources(config, &schemas)?; - let (csr, node_index) = if use_csr { + let (csr, node_index) = if options.use_csr { ( - build_csr_entries(config, &sources, &schemas, previous, fold_csr_deltas).await?, + build_csr_entries( + config, + &sources, + &schemas, + previous, + fold_csr_deltas, + options.persist_csr, + ) + .await?, build_node_index(config, &sources, previous).await?, ) } else { @@ -749,12 +796,18 @@ async fn build_key_statistics( /// snapshot; node-table changes never invalidate it, since the CSR stores raw /// target ids. Relationship types with non-integer endpoints get no entry and /// queries over them use the join path. +/// +/// With `persist`, a Lance relationship dataset's stored CSR is loaded instead of the +/// first build when it can be brought up to date, and every full build is stored. +/// Later rebuilds don't look for a file: the stored base is the one the engine already +/// had, whose delta just grew too large. async fn build_csr_entries( config: &GraphConfig, sources: &HashMap, schemas: &HashMap, previous: Option<&EngineState>, fold_deltas: bool, + persist: bool, ) -> Result> { let mut entries = HashMap::new(); for (_, mapping) in config.iter_relationship_mappings() { @@ -797,11 +850,27 @@ async fn build_csr_entries( } } + if persist && previous_entry.is_none() { + if let Some(entry) = load_stored_csr(source, src.name(), dst.name()).await { + entries.insert(rel_type.clone(), entry); + continue; + } + } + let started = std::time::Instant::now(); // Two projected scans of the endpoint columns, one per direction. let scan = || source.edge_stream(src.name(), dst.name()); let outgoing = Adjacency::build(scan().await?, src.name(), dst.name()).await?; let incoming = Adjacency::build(scan().await?, dst.name(), src.name()).await?; + let build_ms = started.elapsed().as_secs_f64() * 1e3; + if let (true, EngineSource::Lance { dataset, .. }) = (persist, source) { + // A failed write (read-only dataset or credentials) only costs the next start a build. + if let Err(e) = + csr_store::save(dataset, src.name(), dst.name(), &outgoing, &incoming).await + { + log::warn!("Could not store the CSR of {rel_type}: {e}"); + } + } entries.insert( rel_type.clone(), CsrEntry { @@ -810,7 +879,7 @@ async fn build_csr_entries( incoming: Arc::new(incoming), edges: source.edge_rows(), base_version: source.lance_version(), - build_ms: started.elapsed().as_secs_f64() * 1e3, + build_ms, update: if previous_entry.is_some() { "rebuild" } else { @@ -836,11 +905,25 @@ async fn build_csr_entries( /// Building the CSR (`use_csr`) is CPU-bound work inside an async function: in an /// application with its own async runtime, construct and refresh the engine on a /// blocking thread (for example `tokio::task::spawn_blocking` plus `block_on`). +/// Options for [`CypherEngine::with_options`]. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct EngineOptions { + /// Build CSR adjacencies and node indexes; see [`CypherEngine::new`]. + pub use_csr: bool, + /// DataFusion `target_partitions`; see [`CypherEngine::with_target_partitions`]. + pub target_partitions: Option, + /// Store each fully built CSR inside its Lance relationship dataset, at + /// `_lance_csr/.arrow`, and load it instead of building when an engine + /// starts. Changes since the stored version apply as a delta, under + /// the same 25% threshold as `refresh()`. Ignored without `use_csr` and for Arrow + /// input. A failed write is logged and doesn't fail the engine. + pub persist_csr: bool, +} + pub struct CypherEngine { config: GraphConfig, state: RwLock>, - use_csr: bool, - target_partitions: Option, + options: EngineOptions, /// Held for a whole refresh, so a slower refresh can't swap in older data over a newer one. refresh_lock: futures::lock::Mutex<()>, } @@ -880,24 +963,39 @@ impl CypherEngine { sources: HashMap, use_csr: bool, target_partitions: Option, + ) -> Result { + let options = EngineOptions { + use_csr, + target_partitions, + ..Default::default() + }; + Self::with_options(config, sources, options).await + } + + /// Like [`new`](Self::new), with every [`EngineOptions`] setting. + /// + /// # Errors + /// As [`with_target_partitions`](Self::with_target_partitions). + pub async fn with_options( + config: GraphConfig, + sources: HashMap, + options: EngineOptions, ) -> Result { if sources.is_empty() { return Err(config_error("No input datasets provided".to_string())); } - if let Some(partitions) = target_partitions { + if let Some(partitions) = options.target_partitions { if !(1..=MAX_TARGET_PARTITIONS).contains(&partitions) { return Err(config_error(format!( "target_partitions must be between 1 and {MAX_TARGET_PARTITIONS}" ))); } } - let state = - build_engine_state(&config, sources, use_csr, target_partitions, None, false).await?; + let state = build_engine_state(&config, sources, &options, None, false).await?; Ok(Self { config, state: RwLock::new(Arc::new(state)), - use_csr, - target_partitions, + options, refresh_lock: Default::default(), }) } @@ -940,7 +1038,7 @@ impl CypherEngine { .with_parameters(parameters) .with_config(self.config.clone()) .with_fan_out(state.fan_out()); - let mut query = if self.use_csr { + let mut query = if self.options.use_csr { query.with_adjacency(state.adjacency_catalog()) } else { query @@ -1047,15 +1145,8 @@ impl CypherEngine { } } } - let state = build_engine_state( - &self.config, - sources, - self.use_csr, - self.target_partitions, - Some(¤t), - false, - ) - .await?; + let state = + build_engine_state(&self.config, sources, &self.options, Some(¤t), false).await?; *self.state.write().expect("engine state lock poisoned") = Arc::new(state); Ok(()) } @@ -1074,8 +1165,7 @@ impl CypherEngine { let state = build_engine_state( &self.config, current.sources.clone(), - self.use_csr, - self.target_partitions, + &self.options, Some(¤t), true, ) @@ -1084,6 +1174,29 @@ impl CypherEngine { Ok(()) } + /// Delete the CSR files stored for this engine's Lance relationship datasets + /// (see [`EngineOptions::persist_csr`]), returning how many files were deleted. + /// + /// The engine keeps the CSR it has in memory, so queries are unaffected. An + /// engine with `persist_csr` stores a new file at its next full build; create + /// engines without it to stay in memory only. Runs one at a time with `refresh()` + /// and `optimize_csr()`. + pub async fn delete_stored_csr(&self) -> Result { + let _refreshing = self.refresh_lock.lock().await; + let current = self.snapshot(); + let mut deleted = 0; + for (_, mapping) in self.config.iter_relationship_mappings() { + let source = current + .sources + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(&mapping.relationship_type)); + if let Some((_, EngineSource::Lance { dataset, .. })) = source { + deleted += csr_store::delete_all(dataset).await?; + } + } + Ok(deleted) + } + /// CSR adjacency statistics per relationship type (experimental, `use_csr`). /// /// An identical map after `refresh()` means the snapshot itself was unchanged. @@ -1352,4 +1465,197 @@ mod tests { assert_eq!(engine.dataset_versions()["Person"], 2); assert_eq!(engine.node_index_stats()["Person"].rows, 2); } + + async fn write_knows(uri: &str, edges: &[(i64, i64)], mode: WriteMode) { + let schema = Arc::new(Schema::new(vec![ + Field::new("src", DataType::Int64, false), + Field::new("dst", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.0))), + Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.1))), + ], + ) + .unwrap(); + let params = WriteParams { + mode, + max_rows_per_file: 40, + ..Default::default() + }; + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + } + + /// Every `(a, b)` reachable in two hops, sorted, from an engine over `uri`. + async fn two_hops(uri: &str, persist_csr: bool) -> (Vec<(i64, i64)>, CsrStats) { + let config = GraphConfig::builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") + .build() + .unwrap(); + let people = RecordBatch::try_from_iter([( + "id", + Arc::new(Int64Array::from_iter_values(0..60)) as arrow_array::ArrayRef, + )]) + .unwrap(); + let knows = EngineSource::open_lance(uri, None, HashMap::new()) + .await + .unwrap(); + let sources = HashMap::from([ + ("Person".to_string(), EngineSource::Arrow(people)), + ("KNOWS".to_string(), knows), + ]); + let options = EngineOptions { + use_csr: true, + persist_csr, + ..Default::default() + }; + let engine = CypherEngine::with_options(config, sources, options) + .await + .unwrap(); + let batch = engine + .execute( + "MATCH (a:Person)-[:KNOWS]->(:Person)<-[:KNOWS]-(b:Person) \ + RETURN a.id AS a, b.id AS b", + HashMap::new(), + ) + .await + .unwrap(); + let column = |i: usize| { + batch + .column(i) + .as_any() + .downcast_ref::() + .unwrap() + .clone() + }; + let (a, b) = (column(0), column(1)); + let mut rows: Vec<_> = a + .values() + .iter() + .copied() + .zip(b.values().iter().copied()) + .collect(); + rows.sort_unstable(); + (rows, engine.csr_stats()["KNOWS"].clone()) + } + + fn stored_files(uri: &str) -> Vec { + let mut names: Vec = + std::fs::read_dir(std::path::Path::new(uri).join("_lance_csr")) + .map(|entries| { + entries + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + names.sort(); + names + } + + /// A stored CSR is loaded in place of a build, brought up to date with later + /// appends and deletes, ignored when corrupt, and replaced by `optimize_csr()`; + /// every answer matches an engine that builds from scratch. + #[tokio::test] + async fn persisted_csr_loads_and_catches_up() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("knows.lance").display().to_string(); + // Duplicate edges and a self-loop, over three fragments. + let mut edges: Vec<(i64, i64)> = (0..100).map(|i| (i % 50, (i * 7) % 60)).collect(); + edges.extend([(3, 3), (3, 21), (3, 21)]); + write_knows(&uri, &edges, WriteMode::Create).await; + + let (expected, stats) = two_hops(&uri, false).await; + assert_eq!(stats.update, "build"); + assert!( + stored_files(&uri).is_empty(), + "nothing is stored without persist_csr" + ); + + let (rows, stats) = two_hops(&uri, true).await; + assert_eq!((rows == expected, stats.update), (true, "build")); + assert_eq!(stored_files(&uri), ["1.arrow"]); + + let (rows, stats) = two_hops(&uri, true).await; + assert_eq!((rows == expected, stats.update), (true, "loaded")); + assert_eq!((stats.edges, stats.base_version), (103, Some(1))); + + // An append and a delete since the stored version become a delta. + write_knows(&uri, &[(1, 2), (2, 1), (40, 3)], WriteMode::Append).await; + let mut ds = Dataset::open(&uri).await.unwrap(); + ds.delete("src = 3 AND dst = 21").await.unwrap(); + let (expected, _) = two_hops(&uri, false).await; + let (rows, stats) = two_hops(&uri, true).await; + assert_eq!((rows == expected, stats.update), (true, "loaded")); + assert_eq!( + ( + stats.delta_edges, + stats.tombstones, + stats.version, + stats.base_version + ), + (3, 3, Some(3), Some(1)) + ); + assert_eq!( + stored_files(&uri), + ["1.arrow"], + "a delta doesn't store a new base" + ); + + // A corrupt file is ignored: the engine builds and stores the current version. + std::fs::write( + dir.path().join("knows.lance/_lance_csr/1.arrow"), + b"not arrow", + ) + .unwrap(); + let (rows, stats) = two_hops(&uri, true).await; + assert_eq!((rows == expected, stats.update), (true, "build")); + assert_eq!(stored_files(&uri), ["3.arrow"], "older files are pruned"); + + // optimize_csr() stores the folded base of a loaded delta. + write_knows(&uri, &[(5, 6)], WriteMode::Append).await; + let config = GraphConfig::builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") + .build() + .unwrap(); + let knows = EngineSource::open_lance(&uri, None, HashMap::new()) + .await + .unwrap(); + let options = EngineOptions { + use_csr: true, + persist_csr: true, + ..Default::default() + }; + let engine = CypherEngine::with_options( + config, + HashMap::from([("KNOWS".to_string(), knows)]), + options, + ) + .await + .unwrap(); + assert_eq!(engine.csr_stats()["KNOWS"].delta_edges, 1); + engine.optimize_csr().await.unwrap(); + let stats = &engine.csr_stats()["KNOWS"]; + assert_eq!((stats.update, stats.base_version), ("rebuild", Some(4))); + assert_eq!(stored_files(&uri), ["4.arrow"]); + let (rows, stats) = two_hops(&uri, true).await; + assert_eq!(stats.update, "loaded"); + assert_eq!(rows, two_hops(&uri, false).await.0); + + // Deleting the stored files leaves the engine's CSR in place. + assert_eq!(engine.delete_stored_csr().await.unwrap(), 1); + assert!(stored_files(&uri).is_empty()); + assert!(!dir.path().join("knows.lance/_lance_csr").exists()); + assert_eq!(engine.csr_stats()["KNOWS"].edges, 104); + assert_eq!(engine.delete_stored_csr().await.unwrap(), 0); + assert_eq!(two_hops(&uri, true).await.1.update, "build"); + } } diff --git a/crates/lance-csr/src/lib.rs b/crates/lance-csr/src/lib.rs index 5c9b399..7e86159 100644 --- a/crates/lance-csr/src/lib.rs +++ b/crates/lance-csr/src/lib.rs @@ -52,6 +52,7 @@ pub mod case_insensitive; pub mod config; pub mod csr_expand; pub mod csr_index; +mod csr_store; mod csr_traversal; pub mod datafusion_planner; pub mod engine; diff --git a/docs/csr.md b/docs/csr.md index d6ed058..bc12cd0 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -271,7 +271,7 @@ print(engine.csr_stats()["KNOWS"]) | `delta_edges` | Appended edges held outside the base CSR since its last full build. | | `tombstones` | Removed edges masked out since the last full build. | | `unindexed_edges` | Live edges outside the base CSR: appended since its last full build and not since removed. Lance index statistics call this `num_unindexed_rows`. | -| `update` | How the last build, `refresh()` or [`optimize_csr()`](#folding-the-delta-in) produced it: `build` (first), `rebuild` (full), `delta` (changes applied) or `reused` (no edge change). | +| `update` | How the last build, `refresh()` or [`optimize_csr()`](#folding-the-delta-in) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). | | `version` | The relationship dataset's Lance version, or `None` for PyArrow input. | | `base_version` | The version of the last full build, or `None` for PyArrow input. It trails `version` while deltas apply. | | `build_ms` | Wall time of that build or delta, both directions. | @@ -403,6 +403,72 @@ print(stats["update"], stats["unindexed_edges"]) # rebuild 0 `optimize_csr()` doesn't move to newer data, so call `refresh()` first. Like `refresh()`, it builds the replacement in full before swapping it in, and the two run one at a time. +## Storing the CSR + +Every engine builds its CSR from a full scan of the relationship tables, even when the data hasn't changed since +the last run. With `persist_csr=True`, an engine over Lance datasets stores each CSR it builds and loads it on the +next start instead: + +```python +def open_engine(): + lance_datasets = {"Person": lance.dataset("person.lance"), "KNOWS": lance.dataset("knows.lance")} + return CypherEngine(config, lance_datasets, use_csr=True, persist_csr=True) + +print(open_engine().csr_stats()["KNOWS"]["update"]) # build: built, then stored +print(open_engine().csr_stats()["KNOWS"]["update"]) # loaded: read from the file +``` + +Each relationship dataset keeps its CSR in its own directory, one file per dataset version: + +```text +knows.lance/ +├── _versions/ data/ _deletions/ _indices/ _transactions/ +└── _lance_csr/ + └── 42.arrow +``` + +The file belongs to the table: dropping or copying the dataset drops or copies it, and Lance's +`cleanup_old_versions()` leaves the folder alone, since it only deletes the data, index, deletion and transaction +files it recognises. + +- **Changes since the stored version** apply as a delta, read from the new fragments only, exactly as + [`refresh()`](#keeping-the-csr-current) applies them. `update` says `loaded` and `base_version` names the + stored version. That holds after compaction and `cleanup_old_versions()` too. +- **Past the 25% threshold**, the engine rebuilds the CSR and stores the new version. Every full build is stored, + including those from `refresh()` and [`optimize_csr()`](#folding-the-delta-in), and older files are deleted + once a newer one is written. +- **A file that can't be used** (corrupt, written for other endpoint columns or by another format) is ignored and + rebuilt, so a bad file costs a build, never a wrong answer. +- **A dataset that can't be written**, because of read-only storage or credentials, is built as without the option, + and nothing is stored. Rust applications see the failed write as a `log` warning. +- **PyArrow inputs** are never stored. That includes Lance data loaded into memory with `to_table()` first. + +To go back to building in memory only, create engines without `persist_csr` and delete the stored files once with +[`delete_stored_csr()`][lance_csr.CypherEngine.delete_stored_csr]. It removes the `_lance_csr` folder of every +relationship dataset the engine reads and returns how many files it deleted; the engine keeps the CSR it already +holds, so queries carry on as before: + +```python +print(open_engine().delete_stored_csr()) # 1 +``` + +!!! note "Cost" + + The file holds the base CSR's arrays, compressed with zstd: on SF1 about 9 bytes per edge, against the + 32 bytes per edge (plus 16 per source key in each direction) the CSR takes in memory. Node indexes are still + built at start-up. On the GDC SF1 graph read lazily (17.3 million edges over 23 relationship types), measured + on an Apple M5: + + | | Engine start | Peak memory | + | --- | ---: | ---: | + | Without `persist_csr` | 1.55 s | 1.5 to 1.7 GiB | + | First start, building and storing 150 MiB | 2.9 s | 1.5 to 1.7 GiB | + | Later starts, loading | 0.58 s | 1.25 to 1.3 GiB | + + Compressing adds about a second to the first start, and decompressing costs a load about 0.4 s. Query times + don't change. `python/benchmarks/csr_persist.py` measures the same cases on a synthetic graph, including a load + with a delta. + ## Should I turn it on? | Situation | Recommendation | diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md index 5131e99..3b7c92f 100644 --- a/docs/internals/indexes.md +++ b/docs/internals/indexes.md @@ -95,3 +95,30 @@ for a full rebuild. The base and delta map onto Lance's own index design: the delta plays the part of Lance's unindexed rows, and `optimize_csr()` the part of `optimize_indices()`. [Folding the delta in](../csr.md#folding-the-delta-in) has the full comparison. + +## Stored CSR files + +With `persist_csr=True` (see [Storing the CSR](../csr.md#storing-the-csr)), `csr_store.rs` writes the base of both +adjacencies after every full build of a Lance relationship dataset, to `_lance_csr/.arrow` at the dataset's +root. It's one Arrow IPC file holding one record batch, one row per edge in CSR position order: + +| Column | Type | Holds | +| --- | --- | --- | +| `out_source`, `in_source` | run-end encoded `Int64` | Source keys; the run ends are the CSR offsets | +| `out_neighbor`, `in_neighbor` | `UInt64` | Raw target keys | +| `out_edge_row`, `in_edge_row` | `UInt64` | Edge row addresses | + +The schema metadata records the format, the dataset version, the endpoint column names, their Lance field ids and +types, and the fragments the base was built from. The delta is never stored. + +When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as a refresh diffs two versions (see +[How a refresh diffs Lance versions](#how-a-refresh-diffs-lance-versions)), so the file's own version doesn't need to +exist any more. An empty diff uses the file as it is; otherwise the changes become the delta, under the same 25% +threshold. Rebuilds after start-up (from `refresh()` or `optimize_csr()`) don't look for a file, since the stored base +is the one the engine already holds, but they do store the new one. + +The arrays are compressed with zstd, Arrow IPC's own buffer compression, which made the SF1 files 4.7 times smaller +(711 to 150 MiB). `CsrIndex` keeps its arrays as Arrow `ScalarBuffer`s, so a loaded file's arrays are decompressed +straight into the buffers the CSR uses; only the source-key map is rebuilt, one insert per source. Decompression is +single-threaded, about 0.4 s of a load at SF1. Writes go through the dataset's object store with `put_if_absent`, which +publishes the whole file at once on local disk and object storage; the compressed file is encoded in memory first. diff --git a/python/benchmarks/csr_persist.py b/python/benchmarks/csr_persist.py new file mode 100644 index 0000000..844069e --- /dev/null +++ b/python/benchmarks/csr_persist.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Engine start-up with a stored CSR (``persist_csr=True``) against building it. + +For each edge count, writes a deterministic random graph to Lance (nodes = +edges / AVG_DEGREE), then runs each case in a fresh subprocess on its own copy +of the relationship dataset: + +- ``off``: ``persist_csr=False``, today's build; +- ``first``: ``persist_csr=True`` with nothing stored, so a build plus the write; +- ``loaded``: ``persist_csr=True`` with the file from ``first``; +- ``loaded_delta``: as ``loaded``, after appending 1% more edges and deleting about + 1% of the rows, so the changes since the stored version apply as a delta. + +Each case reports construction time, the CSR's ``build_ms`` and ``update``, peak +RSS, the size of ``_lance_csr/`` and warm latency of a few queries, whose row +counts and checksums must match across cases. + + python benchmarks/csr_persist.py --data /tmp/csr-persist --edges 1000000 10000000 +""" + +import argparse +import json +import resource +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from csr_expand import AVG_DEGREE, generate, summary + +CASES = ("off", "first", "loaded", "loaded_delta") +QUERIES = { + "point": "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 42 RETURN b.id", + "two_hop": ( + "MATCH (a:Person)-[:KNOWS]->(:Person)-[:KNOWS]->(c:Person) " + "WHERE a.id < 100 RETURN count(c) AS n" + ), + "all_edges": "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN count(*) AS n", +} + + +def peak_rss_mib() -> float: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return (peak if sys.platform == "darwin" else peak * 1024) / 2**20 + + +def folder_mib(path: Path) -> float: + return sum(f.stat().st_size for f in path.glob("*")) / 2**20 if path.exists() else 0 + + +def child(root: Path, knows: Path, case: str, repeats: int) -> dict: + import lance + import pyarrow.compute as pc + from lance_csr import CypherEngine, GraphConfig + + config = ( + GraphConfig.builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") + .build() + ) + datasets = { + "Person": lance.dataset(root / "person.lance"), + "KNOWS": lance.dataset(knows), + } + kwargs = {} if case == "off" else {"persist_csr": True} + start = time.perf_counter() + engine = CypherEngine(config, datasets, use_csr=True, **kwargs) + out = {"case": case, "construct_ms": (time.perf_counter() - start) * 1e3} + stats = engine.csr_stats()["KNOWS"] + out |= {k: stats[k] for k in ("update", "build_ms", "edges", "delta_edges")} + out["node_index_build_ms"] = engine.node_index_stats()["Person"]["build_ms"] + out["peak_rss_mib"] = peak_rss_mib() + out["stored_mib"] = folder_mib(knows / "_lance_csr") + for name, query in QUERIES.items(): + result = engine.execute(query) + check = {"rows": result.num_rows, "checksum": pc.sum(result.column(0)).as_py()} + samples = [] + for _ in range(repeats): + t = time.perf_counter() + engine.execute(query) + samples.append(time.perf_counter() - t) + out[name] = {**check, **summary(samples)} + return out + + +def change(knows: Path) -> None: + """Append 1% more edges and delete about 1% of the rows.""" + import lance + import numpy as np + import pyarrow as pa + + ds = lance.dataset(knows) + rows = ds.count_rows() + nodes = rows // AVG_DEGREE + rng = np.random.default_rng(11) + added = pa.table( + { + "src": rng.integers(0, nodes, rows // 100, dtype=np.int64), + "dst": rng.integers(0, nodes, rows // 100, dtype=np.int64), + } + ) + lance.write_dataset(added, knows, mode="append") + lance.dataset(knows).delete(f"src < {nodes // 100}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--data", type=Path, required=True) + parser.add_argument("--edges", type=int, nargs="+", default=[1_000_000]) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--cases", nargs="+", choices=CASES, default=list(CASES)) + parser.add_argument("--child", nargs=3, metavar=("ROOT", "KNOWS", "CASE")) + args = parser.parse_args() + + if args.child: + root, knows, case = args.child + print(json.dumps(child(Path(root), Path(knows), case, args.repeats))) + return + + for edges in args.edges: + root = args.data / f"e{edges}" + root.mkdir(parents=True, exist_ok=True) + generate(root, edges) + # `first` writes into this copy; `loaded` and `loaded_delta` read it back. + knows = root / "knows-persist.lance" + shutil.rmtree(knows, ignore_errors=True) + shutil.copytree(root / "knows.lance", knows) + results = {} + for case in args.cases: + if case == "loaded_delta": + change(knows) + target = root / "knows.lance" if case == "off" else knows + cmd = [sys.executable, __file__, "--data", str(args.data)] + cmd += ["--repeats", str(args.repeats), "--child", str(root), str(target), case] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + print(json.dumps({"edges": edges, "case": case, "error": proc.stderr[-2000:]})) + continue + results[case] = json.loads(proc.stdout.strip().splitlines()[-1]) + print(json.dumps({"edges": edges, **results[case]}), flush=True) + same = [c for c in ("off", "first", "loaded") if c in results] + for name in QUERIES: + answers = {c: (results[c][name]["rows"], results[c][name]["checksum"]) for c in same} + assert len(set(answers.values())) <= 1, (edges, name, answers) + + +if __name__ == "__main__": + main() diff --git a/python/python/lance_csr/_internal/graph.pyi b/python/python/lance_csr/_internal/graph.pyi index a82cc84..b636099 100644 --- a/python/python/lance_csr/_internal/graph.pyi +++ b/python/python/lance_csr/_internal/graph.pyi @@ -34,7 +34,7 @@ class _CsrStats(TypedDict): delta_edges: int tombstones: int unindexed_edges: int - update: Literal["build", "rebuild", "delta", "reused"] + update: Literal["build", "rebuild", "loaded", "delta", "reused"] version: int | None base_version: int | None build_ms: float @@ -114,6 +114,7 @@ class CypherEngine: use_csr: bool = False, *, target_partitions: int | None = None, + persist_csr: bool = False, ) -> None: ... def execute( self, @@ -134,6 +135,7 @@ class CypherEngine: ) -> str: ... def refresh(self) -> None: ... def optimize_csr(self) -> None: ... + def delete_stored_csr(self) -> int: ... def node_index_stats(self) -> dict[str, _NodeIndexStats]: ... def csr_stats(self) -> dict[str, _CsrStats]: ... def dataset_versions(self) -> dict[str, int]: ... diff --git a/python/python/lance_csr/_types.py b/python/python/lance_csr/_types.py index 34a9c5a..6dcfdfe 100644 --- a/python/python/lance_csr/_types.py +++ b/python/python/lance_csr/_types.py @@ -54,7 +54,7 @@ class CsrStats(TypedDict): delta_edges: int tombstones: int unindexed_edges: int - update: Literal["build", "rebuild", "delta", "reused"] + update: Literal["build", "rebuild", "loaded", "delta", "reused"] version: int | None base_version: int | None build_ms: float diff --git a/python/python/lance_csr/cypher.py b/python/python/lance_csr/cypher.py index 0130897..1c8cefe 100644 --- a/python/python/lance_csr/cypher.py +++ b/python/python/lance_csr/cypher.py @@ -75,13 +75,23 @@ class CypherEngine: Keyword-only. A target from 1 to 1024 for DataFusion query partitioning; `None` keeps DataFusion's default. `refresh` preserves it. It is not a thread or memory limit. See [Query parallelism](lance.md#query-parallelism). + persist_csr : bool + Keyword-only, experimental, needs `use_csr=True`. Store each relationship + dataset's CSR inside the dataset's own directory, at + `_lance_csr/.arrow`, and load it instead of building when an engine + starts. Changes since the stored version apply as a delta under the same 25% + threshold as `refresh`; past it, the CSR is rebuilt and stored again. Lance's + `cleanup_old_versions` leaves the folder alone. PyArrow inputs aren't stored, + and a dataset that can't be written (read-only storage or credentials) is + built as without it. See [Storing the CSR](csr.md#storing-the-csr). Raises ------ ValueError If `datasets` is empty, a configured key or endpoint column is missing, a Lance - dataset was opened through a namespace client and can't be reopened by URI, or - `target_partitions` is outside 1 to 1024. + dataset was opened through a namespace client and can't be reopened by URI, + `target_partitions` is outside 1 to 1024, or `persist_csr` is set without + `use_csr`. TypeError If `config` isn't a `GraphConfig`, a dataset has an unsupported type, or `target_partitions` is a boolean or another non-integer other than `None`. @@ -118,12 +128,14 @@ def __init__[DatasetT: Dataset]( use_csr: bool = False, *, target_partitions: int | None = None, + persist_csr: bool = False, ) -> None: self._inner = _graph.CypherEngine( inner(config, GraphConfig, "config"), datasets, use_csr, target_partitions=target_partitions, + persist_csr=persist_csr, ) def __repr__(self) -> str: @@ -278,6 +290,24 @@ def optimize_csr(self) -> None: """ self._inner.optimize_csr() + def delete_stored_csr(self) -> int: + """Delete the CSR files stored with `persist_csr=True` (experimental). + + Removes the `_lance_csr` folder of every Lance relationship dataset this + engine reads, whether or not the engine itself was created with + `persist_csr`. The engine keeps the CSR it already holds in memory, so queries + aren't affected. To go back to in-memory builds only, create later engines + without `persist_csr`: an engine with it stores a new file at its next full + build. Runs one at a time with `refresh` and `optimize_csr`. See + [Storing the CSR](csr.md#storing-the-csr). + + Returns + ------- + int + The number of files deleted; 0 when nothing was stored. + """ + return self._inner.delete_stored_csr() + def node_index_stats(self) -> dict[str, NodeIndexStats]: """Statistics for each node index (experimental; needs `use_csr=True`). @@ -311,8 +341,9 @@ def csr_stats(self) -> dict[str, CsrStats]: last full build and not since removed. - `update`: how the last refresh or [`optimize_csr`][lance_csr.CypherEngine.optimize_csr] produced it: - `"build"` (first), `"rebuild"` (full), `"delta"` (changes applied) or - `"reused"` (no edge change). + `"build"` (first), `"rebuild"` (full), `"loaded"` (from a file stored with + `persist_csr=True`, with any changes since applied as a delta), `"delta"` + (changes applied) or `"reused"` (no edge change). - `version`: the relationship dataset version it reflects (`None` for PyArrow). - `base_version`: the relationship dataset version of the last full diff --git a/python/python/tests/test_engine_csr_persist.py b/python/python/tests/test_engine_csr_persist.py new file mode 100644 index 0000000..b45bbd6 --- /dev/null +++ b/python/python/tests/test_engine_csr_persist.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Stored CSR files (`persist_csr=True`). + +An engine that loads a stored CSR must return exactly what a join-path engine +returns, after appends, deletes, compaction and cleanup of old versions. +""" + +import os +import sys +from datetime import timedelta + +import lance +import pyarrow as pa +import pytest +from lance_csr import CypherEngine, GraphConfig + +CONFIG = ( + GraphConfig.builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") + .build() +) +N = 40 +PEOPLE = pa.table({"id": list(range(N)), "name": [f"p{i}" for i in range(N)]}) +QUERIES = [ + "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN a.id, b.id", + "MATCH (a:Person)<-[:KNOWS]-(b:Person) RETURN a.id, b.name", + "MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person) RETURN count(*) AS n", + "MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN a.id, b.id, r.since", +] + + +def edges(pairs, start=0): + src, dst = zip(*pairs) + return pa.table( + { + "src": pa.array(src, pa.int64()), + "dst": pa.array(dst, pa.int64()), + "since": list(range(start, start + len(pairs))), + } + ) + + +# 200 edges in 5 fragments, with duplicates. +BASE = [(i % N, (i * 7 + 1) % N) for i in range(190)] + [(i, i + 1) for i in range(10)] + + +def rows(table): + return sorted(table.to_pylist(), key=lambda r: [str(v) for v in r.values()]) + + +def stored(uri): + folder = uri / "_lance_csr" + return sorted(os.listdir(folder)) if folder.exists() else [] + + +@pytest.fixture +def knows(tmp_path): + uri = tmp_path / "knows.lance" + lance.write_dataset(edges(BASE), uri, max_rows_per_file=40) + return uri + + +def engine(uri, **kwargs): + datasets = {"Person": PEOPLE, "KNOWS": lance.dataset(uri)} + return CypherEngine(CONFIG, datasets, use_csr=True, persist_csr=True, **kwargs) + + +def check(persisted, uri): + join = CypherEngine(CONFIG, {"Person": PEOPLE, "KNOWS": lance.dataset(uri)}) + for query in QUERIES: + assert rows(persisted.execute(query)) == rows(join.execute(query)), query + + +def test_stored_csr_is_loaded(knows): + first = engine(knows) + assert first.csr_stats()["KNOWS"]["update"] == "build" + assert stored(knows) == ["1.arrow"] + second = engine(knows) + s = second.csr_stats()["KNOWS"] + assert (s["update"], s["base_version"], s["edges"]) == ("loaded", 1, len(BASE)) + check(second, knows) + + +def test_stored_csr_survives_compaction_and_cleanup(knows): + engine(knows) + lance.write_dataset(edges([(3, 4), (4, 3)], 4000), knows, mode="append") + lance.write_dataset(edges([(5, 6), (6, 5)], 4002), knows, mode="append") + lance.dataset(knows).optimize.compact_files(target_rows_per_fragment=40) + lance.dataset(knows).delete("since = 0 OR since = 4003") + # Version 1, the stored file's version, is gone after this. + lance.dataset(knows).cleanup_old_versions( + older_than=timedelta(0), delete_unverified=True + ) + assert stored(knows) == ["1.arrow"] + loaded = engine(knows) + s = loaded.csr_stats()["KNOWS"] + assert (s["update"], s["base_version"]) == ("loaded", 1) + assert s["delta_edges"] > 0 and s["tombstones"] > 0 + check(loaded, knows) + + +def test_rebuild_stores_a_new_base(knows): + engine(knows) + # More than 25% of the base appended: the next start rebuilds and stores again. + lance.write_dataset(edges(BASE[:60], 5000), knows, mode="append") + rebuilt = engine(knows) + assert rebuilt.csr_stats()["KNOWS"]["update"] == "build" + assert stored(knows) == ["2.arrow"] + check(rebuilt, knows) + assert engine(knows).csr_stats()["KNOWS"]["update"] == "loaded" + + +@pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, + reason="needs POSIX permissions", +) +def test_read_only_dataset_still_builds(knows): + os.chmod(knows, 0o555) + try: + read_only = engine(knows) + finally: + os.chmod(knows, 0o755) + assert read_only.csr_stats()["KNOWS"]["update"] == "build" + assert stored(knows) == [] + check(read_only, knows) + + +def test_delete_stored_csr(knows): + persisted = engine(knows) + assert stored(knows) == ["1.arrow"] + assert persisted.delete_stored_csr() == 1 + assert not (knows / "_lance_csr").exists() + check(persisted, knows) # the in-memory CSR stays + assert persisted.delete_stored_csr() == 0 + in_memory = CypherEngine( + CONFIG, {"Person": PEOPLE, "KNOWS": lance.dataset(knows)}, use_csr=True + ) + assert in_memory.csr_stats()["KNOWS"]["update"] == "build" + assert stored(knows) == [] + + +def test_pyarrow_input_is_not_stored(): + datasets = {"Person": PEOPLE, "KNOWS": edges(BASE)} + e = CypherEngine(CONFIG, datasets, use_csr=True, persist_csr=True) + assert e.csr_stats()["KNOWS"]["update"] == "build" + + +def test_persist_csr_needs_use_csr(knows): + with pytest.raises(ValueError, match="persist_csr requires use_csr"): + CypherEngine(CONFIG, {"KNOWS": lance.dataset(knows)}, persist_csr=True) From 94769f1d1bf3795db2b7e1c0fa36c06de6509e31 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:09:32 -0500 Subject: [PATCH 2/9] perf(csr): pack edge rows in blocks and store the csr packed Each CSR slot held its edge row as a raw 64-bit Lance row address. Edge rows are now packed in blocks of 128 values, each stored as its offset from the block's minimum in as few bits as the block needs, with every position still directly readable. On GDC SF1 that cuts the CSR's arrays from 613 to 419 MiB, and the whole in-memory CSR from 926 to 732 MiB. Plain hops never unpack edge rows. Neighbours stay a plain array: packing them too slowed lookup-heavy queries (custom Q30 by about 22%) for a smaller saving. Stored CSR files now hold the arrays as they sit in memory (format 2, still zstd compressed), so a load only decompresses and checks them: 134 MiB at SF1, and engine start drops from 0.58 to 0.46 s. The CSR guide now contrasts refresh() and optimize_csr() in a table, and its memory figures are measured rather than estimated. --- crates/lance-csr/src/csr_expand.rs | 52 +++-- crates/lance-csr/src/csr_index.rs | 310 +++++++++++++++++++++++++++-- crates/lance-csr/src/csr_store.rs | 158 +++++++-------- crates/lance-csr/src/engine.rs | 2 + docs/csr.md | 48 +++-- docs/internals/indexes.md | 47 +++-- docs/lance.md | 2 +- python/README.md | 2 +- 8 files changed, 461 insertions(+), 160 deletions(-) diff --git a/crates/lance-csr/src/csr_expand.rs b/crates/lance-csr/src/csr_expand.rs index 5f24dec..6fac611 100644 --- a/crates/lance-csr/src/csr_expand.rs +++ b/crates/lance-csr/src/csr_expand.rs @@ -100,9 +100,10 @@ use crate::error::{GraphError, Result}; /// sorted offsets into each source's base-plus-appended neighbours. Offsets let /// readers skip deleted occurrences without copying per-source deletion state. /// Identical edges remain distinct occurrences. Each neighbour carries the row its -/// edge came from (a Lance row address, or a row index for in-memory input), so the -/// base holds 16 bytes per edge: relationship columns can be fetched by row, and -/// deleted edges are found by row without reading the old dataset version. +/// edge came from (a Lance row address, or a row index for in-memory input), so +/// relationship columns can be fetched by row, and deleted edges are found by row +/// without reading the old dataset version. Edge rows are packed (see +/// [`crate::csr_index::PackedU64`]), so the base holds about 10 bytes per edge, not 16. /// /// # Design note for future changes (read before "optimizing" the delta) /// @@ -220,14 +221,16 @@ impl Adjacency { .map(|(k, v)| (k, Some(v))) .chain(added_only.map(|&k| (k, None))) { - let base_rows = vertex.map_or(&[][..], |v| self.base.csr.edge_rows(v)); + let base_range = vertex.map_or(0..0, |v| self.base.csr.range(v)); + let base_len = base_range.len(); let added_rows = self.added.get(&key).map_or(&[][..], Vec::as_slice); let prior = self.removed.get(&key).map_or(&[][..], Vec::as_slice); let mut prior = prior.iter().copied().peekable(); let mut positions = Vec::new(); - let rows = base_rows - .iter() - .copied() + let rows = self + .base + .csr + .edge_rows_at(base_range) .chain(added_rows.iter().map(|&(_, r)| r)); for (position, row) in rows.enumerate() { if prior.peek() == Some(&position) { @@ -239,7 +242,7 @@ impl Adjacency { if !positions.is_empty() { next.removed_count += positions.len() as u64; next.removed_added_count += - positions.iter().filter(|&&p| p >= base_rows.len()).count() as u64; + positions.iter().filter(|&&p| p >= base_len).count() as u64; let tombstones = next.removed.entry(key).or_default(); tombstones.extend(positions); tombstones.sort_unstable(); @@ -286,12 +289,13 @@ impl Adjacency { self.neighbors_from(key, &mut 0).for_each(f); } - /// Neighbours and their edge rows in the base. + /// Positions of `key`'s neighbours in the base. #[inline(always)] - fn base_slots(&self, key: i64) -> (&[u64], &[u64]) { - self.base.dense.get(&key).map_or((&[], &[]), |&v| { - (self.base.csr.neighbors(v), self.base.csr.edge_rows(v)) - }) + fn base_range(&self, key: i64) -> std::ops::Range { + self.base + .dense + .get(&key) + .map_or(0..0, |&v| self.base.csr.range(v)) } /// Resume at an occurrence offset, borrowing all neighbour and deletion storage. @@ -315,17 +319,27 @@ impl Adjacency { key: i64, offset: &'a mut usize, ) -> impl Iterator> + 'a { - let (base, base_rows) = self.base_slots(key); + let base = self.base_range(key); let added = self.added.get(&key).map_or(&[][..], Vec::as_slice); let removed = self.removed.get(&key).map_or(&[][..], Vec::as_slice); let mut removed = &removed[removed.partition_point(|&position| position < *offset)..]; let added_offset = offset.saturating_sub(base.len()); - let base_start = (*offset).min(base.len()); - let base_rows = &base_rows[base_start..]; - base[base_start..] + let rest = base.start + (*offset).min(base.len())..base.end; + let csr = &self.base.csr; + // Never advanced without `ROWS`, so plain hops don't unpack edge rows. + let mut base_rows = csr.edge_rows_at(if ROWS { rest.clone() } else { 0..0 }); + csr.neighbors_at(rest) .iter() - .enumerate() - .map(move |(i, &n)| (n, if ROWS { base_rows[i] } else { 0 })) + .map(move |&n| { + ( + n, + if ROWS { + base_rows.next().unwrap_or(0) + } else { + 0 + }, + ) + }) .chain( added .get(added_offset..) diff --git a/crates/lance-csr/src/csr_index.rs b/crates/lance-csr/src/csr_index.rs index c30a05c..1b48e1a 100644 --- a/crates/lance-csr/src/csr_index.rs +++ b/crates/lance-csr/src/csr_index.rs @@ -17,11 +17,227 @@ //! //! For vertex `v`, its neighbors are `neighbors[offsets[v]..offsets[v+1]]`, and //! `edge_rows` at the same positions holds the row each edge came from. +//! +//! Edge rows are stored packed ([`PackedU64`]): in blocks of 128 values, each held as +//! its offset from the block's minimum in as few bits as the block needs. They are +//! Lance row addresses, so a single width for the whole array would waste most of the +//! saving; per-block widths keep any position directly readable, which traversal +//! needs to resume part-way through a list. Neighbours stay a plain array: every hop +//! reads them, and packing them slowed lookup-heavy queries (custom Q30 by about 22%) +//! for a smaller saving than edge rows give. use std::collections::HashMap; +use std::ops::Range; use arrow::buffer::ScalarBuffer; +/// Values per block of a [`PackedU64`]. +const BLOCK: usize = 128; + +/// A `u64` array packed block by block, with direct access to any position. +/// +/// Each block of [`BLOCK`] values stores its minimum (`bases`) and its start bit +/// and width (`meta`: start bit << 7 | width); the values are bit-packed offsets +/// from the minimum, back to back in `words`. A block whose values are all equal +/// takes no bits at all. +#[derive(Debug, Clone)] +pub struct PackedU64 { + len: usize, + bases: ScalarBuffer, + meta: ScalarBuffer, + /// One spare word at the end, so reading two words never runs past it. + words: ScalarBuffer, +} + +impl PackedU64 { + pub fn pack(values: &[u64]) -> Self { + let blocks = values.len().div_ceil(BLOCK); + let mut bases = Vec::with_capacity(blocks); + let mut meta = Vec::with_capacity(blocks); + let mut widths = Vec::with_capacity(blocks); + let mut bits = 0u64; + for block in values.chunks(BLOCK) { + let min = *block.iter().min().expect("chunks are not empty"); + let max = *block.iter().max().expect("chunks are not empty"); + let width = u64::from(64 - (max - min).leading_zeros()); + bases.push(min); + meta.push(bits << 7 | width); + widths.push(width); + bits += width * block.len() as u64; + } + let mut words = vec![0u64; bits.div_ceil(64) as usize + 1]; + for ((block, &min), (&m, &width)) in values + .chunks(BLOCK) + .zip(&bases) + .zip(meta.iter().zip(&widths)) + { + let mut bit = m >> 7; + for &value in block { + write_bits(&mut words, bit, width as u32, value - min); + bit += width; + } + } + Self { + len: values.len(), + bases: bases.into(), + meta: meta.into(), + words: words.into(), + } + } + + /// A packed array from [`Self::parts`] and its length, or `None` unless every + /// block header is consistent with `len` and the words hold exactly its bits. + pub fn from_parts(len: usize, [bases, meta, words]: [ScalarBuffer; 3]) -> Option { + let blocks = len.div_ceil(BLOCK); + if bases.len() != blocks || meta.len() != blocks { + return None; + } + let mut bits = 0u64; + for (block, &m) in meta.iter().enumerate() { + let (start, width) = split_meta(m); + if start != bits || width > 64 { + return None; + } + bits += u64::from(width) * (len - block * BLOCK).min(BLOCK) as u64; + } + (words.len() as u64 == bits.div_ceil(64) + 1).then_some(Self { + len, + bases, + meta, + words, + }) + } + + /// The block bases, block headers and words, for storing. + pub fn parts(&self) -> [ScalarBuffer; 3] { + [self.bases.clone(), self.meta.clone(), self.words.clone()] + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Value at `index`. + /// + /// # Panics + /// If `index` is out of bounds. + #[inline] + pub fn get(&self, index: usize) -> u64 { + assert!(index < self.len, "index {index} out of bounds"); + let block = index / BLOCK; + let (start, width) = split_meta(self.meta[block]); + let bit = start + (index % BLOCK) as u64 * u64::from(width); + self.bases[block].wrapping_add(read_bits(&self.words, bit, width)) + } + + /// The values at `range`, in order, reading each block's header once. + #[inline] + pub fn iter(&self, range: Range) -> PackedIter<'_> { + assert!(range.end <= self.len, "range {range:?} out of bounds"); + PackedIter { + packed: self, + position: range.start, + end: range.end.max(range.start), + block_end: range.start, + bit: 0, + width: 0, + base: 0, + } + } + + /// Every value, unpacked. + pub fn to_vec(&self) -> Vec { + self.iter(0..self.len).collect() + } + + /// Bytes held by the packed arrays. + pub fn size_bytes(&self) -> usize { + 8 * (self.bases.len() + self.meta.len() + self.words.len()) + } +} + +#[inline(always)] +fn split_meta(meta: u64) -> (u64, u32) { + (meta >> 7, (meta & 0x7f) as u32) +} + +#[inline(always)] +fn read_bits(words: &[u64], bit: u64, width: u32) -> u64 { + if width == 0 { + return 0; + } + let word = (bit >> 6) as usize; + let shift = (bit & 63) as u32; + let mut value = words[word] >> shift; + if shift + width > 64 { + value |= words[word + 1] << (64 - shift); + } + if width < 64 { + value & ((1u64 << width) - 1) + } else { + value + } +} + +fn write_bits(words: &mut [u64], bit: u64, width: u32, value: u64) { + if width == 0 { + return; + } + let word = (bit >> 6) as usize; + let shift = (bit & 63) as u32; + words[word] |= value << shift; + if shift + width > 64 { + words[word + 1] |= value >> (64 - shift); + } +} + +/// Iterator over a range of a [`PackedU64`]; see [`PackedU64::iter`]. +pub struct PackedIter<'a> { + packed: &'a PackedU64, + position: usize, + end: usize, + /// Where the block loaded into `bit`, `width` and `base` ends. + block_end: usize, + bit: u64, + width: u32, + base: u64, +} + +impl Iterator for PackedIter<'_> { + type Item = u64; + + #[inline(always)] + fn next(&mut self) -> Option { + if self.position == self.end { + return None; + } + if self.position == self.block_end { + let block = self.position / BLOCK; + let (start, width) = split_meta(self.packed.meta[block]); + self.width = width; + self.bit = start + (self.position % BLOCK) as u64 * u64::from(width); + self.base = self.packed.bases[block]; + self.block_end = ((block + 1) * BLOCK).min(self.packed.len); + } + let value = read_bits(&self.packed.words, self.bit, self.width); + self.bit += u64::from(self.width); + self.position += 1; + Some(self.base.wrapping_add(value)) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let n = self.end - self.position; + (n, Some(n)) + } +} + +impl ExactSizeIterator for PackedIter<'_> {} + /// In-memory CSR adjacency index for fast neighbor lookup. /// /// Stores graph topology in three arrays: @@ -30,23 +246,22 @@ use arrow::buffer::ScalarBuffer; /// - `edge_rows`, aligned with `neighbors` = the row (a Lance row address, or a /// row index for in-memory input) each edge came from /// -/// The arrays are Arrow buffers, so an index read from a stored file shares the -/// file's memory instead of copying it. +/// Edge rows are packed; see the module docs. #[derive(Debug, Clone)] pub struct CsrIndex { offsets: ScalarBuffer, neighbors: ScalarBuffer, - edge_rows: ScalarBuffer, + edge_rows: PackedU64, } impl CsrIndex { - /// An index from its three arrays, as [`Self::parts`] returns them, or `None` + /// An index from its three arrays, as [`Self::packed`] returns them, or `None` /// if `offsets` is empty, descends or doesn't end at `neighbors.len()`, or /// `edge_rows` and `neighbors` differ in length. - pub fn from_parts( + pub fn from_packed( offsets: ScalarBuffer, neighbors: ScalarBuffer, - edge_rows: ScalarBuffer, + edge_rows: PackedU64, ) -> Option { let well_formed = offsets.first() == Some(&0) && offsets.last() == Some(&(neighbors.len() as u64)) @@ -59,12 +274,15 @@ impl CsrIndex { }) } - /// The offsets, neighbours and edge rows. - pub fn parts(&self) -> (&ScalarBuffer, &ScalarBuffer, &ScalarBuffer) { + /// The offsets, the neighbours and the packed edge rows. + pub fn packed(&self) -> (&ScalarBuffer, &ScalarBuffer, &PackedU64) { (&self.offsets, &self.neighbors, &self.edge_rows) } - fn range(&self, vertex_id: u64) -> std::ops::Range { + /// Positions of a vertex's neighbours; empty for vertices with no edges or + /// beyond the index range. + #[inline] + pub fn range(&self, vertex_id: u64) -> Range { let v = vertex_id as usize; if v >= self.offsets.len() - 1 { return 0..0; @@ -72,21 +290,38 @@ impl CsrIndex { self.offsets[v] as usize..self.offsets[v + 1] as usize } - /// Look up all neighbors of a vertex. Returns an empty slice for vertices - /// with no outgoing edges or vertex IDs beyond the index range. + /// The neighbours at `positions`, a sub-range of one or more [`Self::range`]s. + #[inline] + pub fn neighbors_at(&self, positions: Range) -> &[u64] { + &self.neighbors[positions] + } + + /// The edge rows at `positions`, aligned with [`Self::neighbors_at`]. + #[inline] + pub fn edge_rows_at(&self, positions: Range) -> PackedIter<'_> { + self.edge_rows.iter(positions) + } + + /// All neighbours of a vertex; empty for vertices with no edges or beyond the + /// index range. pub fn neighbors(&self, vertex_id: u64) -> &[u64] { - &self.neighbors[self.range(vertex_id)] + self.neighbors_at(self.range(vertex_id)) } - /// The edge rows of a vertex's neighbors, in the same order as [`Self::neighbors`]. - pub fn edge_rows(&self, vertex_id: u64) -> &[u64] { - &self.edge_rows[self.range(vertex_id)] + /// The edge rows of a vertex's neighbours, in the same order as [`Self::neighbors`]. + pub fn edge_rows(&self, vertex_id: u64) -> PackedIter<'_> { + self.edge_rows_at(self.range(vertex_id)) } /// Return the total number of edges in the index. pub fn num_edges(&self) -> u64 { self.neighbors.len() as u64 } + + /// Bytes held by the offsets, neighbours and edge rows. + pub fn size_bytes(&self) -> usize { + 8 * (self.offsets.len() + self.neighbors.len()) + self.edge_rows.size_bytes() + } } /// Builder for constructing a CSR index from edge data. @@ -156,11 +391,13 @@ impl CsrIndexBuilder { let edge_rows: Vec = self.edges.iter().map(|&(_, _, row)| row).collect(); neighbors.extend(self.edges.iter().map(|&(_, dst, _)| dst)); + // Freed before packing, so the peak holds the unpacked rows or the edges, not both. + drop(std::mem::take(&mut self.edges)); CsrIndex { offsets: offsets.into(), neighbors: neighbors.into(), - edge_rows: edge_rows.into(), + edge_rows: PackedU64::pack(&edge_rows), } } } @@ -199,9 +436,9 @@ mod tests { assert_eq!(idx.neighbors(1), &[2]); assert_eq!(idx.neighbors(2), &[3]); assert_eq!(idx.neighbors(3), &[] as &[u64]); - assert_eq!(idx.edge_rows(0), &[0, 1, 2]); - assert_eq!(idx.edge_rows(2), &[4]); - assert_eq!(idx.edge_rows(99), &[] as &[u64]); + assert_eq!(idx.edge_rows(0).collect::>(), [0, 1, 2]); + assert_eq!(idx.edge_rows(2).collect::>(), [4]); + assert_eq!(idx.edge_rows(99).collect::>(), Vec::::new()); } #[test] @@ -263,6 +500,41 @@ mod tests { assert_eq!(idx.neighbors(1), &[1]); } + #[test] + fn packed_values_round_trip() { + // Constant blocks, full 64-bit spans, values straddling words and a short + // last block. + let mut values: Vec = vec![7; 130]; + values.extend([0, u64::MAX, 1 << 63, 3]); + values.extend((0..300u64).map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> (i % 60))); + values.extend((0..200u64).map(|i| 2_199_023_255_552 + i * 37)); + let packed = PackedU64::pack(&values); + assert_eq!(packed.to_vec(), values); + for (i, &v) in values.iter().enumerate() { + assert_eq!(packed.get(i), v, "position {i}"); + } + for start in [0, 1, 127, 128, 129, 300, values.len()] { + let got: Vec = packed.iter(start..values.len()).collect(); + assert_eq!(got, values[start..], "from {start}"); + } + assert_eq!(packed.iter(5..5).count(), 0); + assert!(packed.size_bytes() < 8 * values.len()); + assert!(PackedU64::pack(&[]).is_empty()); + + // Parts round-trip, and inconsistent ones are refused. + let again = PackedU64::from_parts(values.len(), packed.parts()).unwrap(); + assert_eq!(again.to_vec(), values); + assert!(PackedU64::from_parts(values.len() + 128, packed.parts()).is_none()); + let [bases, meta, words] = packed.parts(); + let short = words.slice(0, words.len() - 1); + assert!( + PackedU64::from_parts(values.len(), [bases.clone(), meta.clone(), short]).is_none() + ); + let mut bad_meta = meta.to_vec(); + bad_meta[1] += 1 << 7; + assert!(PackedU64::from_parts(values.len(), [bases, bad_meta.into(), words]).is_none()); + } + #[test] fn test_parallel_edges() { let idx = CsrIndexBuilder::new() diff --git a/crates/lance-csr/src/csr_store.rs b/crates/lance-csr/src/csr_store.rs index 347fddf..fedb671 100644 --- a/crates/lance-csr/src/csr_store.rs +++ b/crates/lance-csr/src/csr_store.rs @@ -9,23 +9,26 @@ //! index, deletion and transaction files, temporary manifests), so it leaves this //! folder alone. //! -//! One record batch holds both directions, one row per edge in CSR position order: -//! `out_source`/`in_source` are run-end encoded source keys (the run ends are the -//! CSR offsets), next to `*_neighbor` and `*_edge_row`. The schema metadata records -//! the dataset version, the endpoint columns and fields, and the fragments the base -//! was built from, so [`edge_changes_since`] can bring an older file up to date. +//! The file holds the CSR as it sits in memory, so a load neither builds nor packs +//! anything. The schema has one `UInt64` column, `values`, and the file is a +//! sequence of [`ARRAYS`]-many record batches of different lengths, one array each, +//! for the outgoing then the incoming direction: the source keys (`i64` bit-cast) +//! in CSR position order, the offsets, the neighbours, then the bases, block headers +//! and words of the packed edge rows (see [`crate::csr_index::PackedU64`]). Every batch is zstd-compressed. The arrays are +//! not meant to be read on their own; this module is their only reader. //! -//! A file is used only if it parses, its arrays are well formed and its endpoint -//! fields match; anything else is ignored and the CSR is rebuilt. The arrays are -//! zstd-compressed (4.7 times smaller on SF1), so loading decompresses them straight -//! into the CSR's buffers. +//! The schema metadata records the dataset version, the endpoint columns and fields, +//! and the fragments the base was built from, so [`edge_changes_since`] can bring an +//! older file up to date. A file is used only if it parses, every array is well +//! formed and its endpoint fields match; anything else is ignored and the CSR is +//! rebuilt. use std::collections::HashMap; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, AsArray, Int64Array, RecordBatch, RunArray, UInt64Array}; +use arrow::array::{Array, AsArray, RecordBatch, UInt64Array}; use arrow::buffer::{Buffer, ScalarBuffer}; -use arrow::datatypes::{DataType, Field, Int64Type, Schema, UInt64Type}; +use arrow::datatypes::{DataType, Field, Schema, UInt64Type}; use arrow::ipc::convert::fb_to_schema; use arrow::ipc::reader::{read_footer_length, FileDecoder}; use arrow::ipc::root_as_footer; @@ -36,13 +39,16 @@ use lance_table::format::Fragment; use object_store::path::Path; use crate::csr_expand::{endpoint_fields, Adjacency, EndpointFields}; -use crate::csr_index::CsrIndex; +use crate::csr_index::{CsrIndex, PackedU64}; use crate::error::{GraphError, Result}; /// Folder at the dataset root that holds the files. const DIR: &str = "_lance_csr"; /// Bumped whenever the layout changes; files of another format are ignored. -const FORMAT: &str = "1"; +const FORMAT: &str = "2"; +/// Arrays per direction: keys, offsets, neighbours, then the bases, headers and words +/// of the packed edge rows. +const ARRAYS: usize = 6; const KEY_FORMAT: &str = "lance_csr.format"; const KEY_VERSION: &str = "lance_csr.version"; @@ -187,17 +193,6 @@ pub(crate) async fn delete_all(ds: &Dataset) -> Result { Ok(deleted) } -fn source_field(name: &str) -> Field { - Field::new( - name, - DataType::RunEndEncoded( - Arc::new(Field::new("run_ends", DataType::Int64, false)), - Arc::new(Field::new("values", DataType::Int64, true)), - ), - false, - ) -} - fn encode( version: u64, src: &str, @@ -207,6 +202,7 @@ fn encode( outgoing: &Adjacency, incoming: &Adjacency, ) -> Result> { + let failed = |e: arrow::error::ArrowError| store_error(format!("Failed to encode CSR: {e}")); let metadata = HashMap::from([ (KEY_FORMAT.to_string(), FORMAT.to_string()), (KEY_VERSION.to_string(), version.to_string()), @@ -214,48 +210,36 @@ fn encode( (KEY_FIELDS.to_string(), to_json(fields)?), (KEY_FRAGMENTS.to_string(), to_json(fragments)?), ]); - let mut schema_fields = Vec::new(); - let mut columns = Vec::new(); - for (prefix, adjacency) in [("out", outgoing), ("in", incoming)] { - schema_fields.extend([ - source_field(&format!("{prefix}_source")), - Field::new(format!("{prefix}_neighbor"), DataType::UInt64, false), - Field::new(format!("{prefix}_edge_row"), DataType::UInt64, false), - ]); - columns.extend(encode_direction(adjacency)?); - } - let schema = Arc::new(Schema::new_with_metadata(schema_fields, metadata)); - let batch = RecordBatch::try_new(schema.clone(), columns) - .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + let schema = Arc::new(Schema::new_with_metadata( + vec![Field::new("values", DataType::UInt64, false)], + metadata, + )); let options = IpcWriteOptions::default() .try_with_compression(Some(CompressionType::ZSTD)) - .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + .map_err(failed)?; let mut bytes = Vec::new(); - let mut writer = FileWriter::try_new_with_options(&mut bytes, &schema, options) - .and_then(|mut w| w.write(&batch).map(|_| w)) - .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; - writer - .finish() - .map_err(|e| store_error(format!("Failed to encode CSR: {e}")))?; + let mut writer = + FileWriter::try_new_with_options(&mut bytes, &schema, options).map_err(failed)?; + for adjacency in [outgoing, incoming] { + for array in direction_arrays(adjacency) { + let column = Arc::new(UInt64Array::new(array, None)); + let batch = RecordBatch::try_new(schema.clone(), vec![column]).map_err(failed)?; + writer.write(&batch).map_err(failed)?; + } + } + writer.finish().map_err(failed)?; drop(writer); Ok(bytes) } -/// Source keys, neighbours and edge rows of one direction, one row per edge. -fn encode_direction(adjacency: &Adjacency) -> Result<[ArrayRef; 3]> { +/// The [`ARRAYS`] arrays of one direction, shared with the CSR rather than copied +/// (except the keys, collected from the source-key map). +fn direction_arrays(adjacency: &Adjacency) -> [ScalarBuffer; ARRAYS] { let (keys, csr) = adjacency.base_parts(); - let (offsets, neighbors, edge_rows) = csr.parts(); - // Every base source has at least one edge, so `offsets[1..]` are strictly - // ascending: valid run ends, shared with the offsets rather than copied. - let ends = ScalarBuffer::::new(offsets.inner().clone(), 1, keys.len()); - let sources = - RunArray::::try_new(&Int64Array::new(ends, None), &Int64Array::from(keys)) - .map_err(|e| store_error(format!("Failed to encode CSR sources: {e}")))?; - Ok([ - Arc::new(sources), - Arc::new(UInt64Array::new(neighbors.clone(), None)), - Arc::new(UInt64Array::new(edge_rows.clone(), None)), - ]) + let (offsets, neighbors, edge_rows) = csr.packed(); + let keys: Vec = keys.into_iter().map(|k| k as u64).collect(); + let [rb, rm, rw] = edge_rows.parts(); + [keys.into(), offsets.clone(), neighbors.clone(), rb, rm, rw] } fn decode(bytes: Buffer, src: &str, dst: &str) -> Result { @@ -289,45 +273,47 @@ fn decode(bytes: Buffer, src: &str, dst: &str) -> Result { let fragments = from_json(meta(KEY_FRAGMENTS)?)?; let blocks = footer.recordBatches().ok_or_else(|| bad("no batches"))?; - if blocks.len() != 1 { - return Err(bad("expected one batch")); + if blocks.len() != 2 * ARRAYS { + return Err(bad("wrong number of arrays")); } - let block = blocks.get(0); - let length = block.bodyLength() as usize + block.metaDataLength() as usize; - if block.offset() as usize + length > bytes.len() { - return Err(bad("batch out of range")); + let decoder = FileDecoder::new(schema, footer.version()); + let mut arrays = Vec::with_capacity(blocks.len()); + for block in blocks.iter() { + let length = block.bodyLength() as usize + block.metaDataLength() as usize; + if block.offset() < 0 || block.offset() as usize + length > bytes.len() { + return Err(bad("array out of range")); + } + let data = bytes.slice_with_length(block.offset() as usize, length); + let batch = decoder + .read_record_batch(block, &data) + .map_err(ipc)? + .ok_or_else(|| bad("empty batch"))?; + let column = batch.column(0).as_primitive_opt::(); + let column = column + .filter(|c| c.null_count() == 0) + .ok_or_else(|| bad("array type"))?; + arrays.push(column.values().clone()); } - let data = bytes.slice_with_length(block.offset() as usize, length); - let batch = FileDecoder::new(schema, footer.version()) - .read_record_batch(block, &data) - .map_err(ipc)? - .ok_or_else(|| bad("empty batch"))?; + let incoming = arrays.split_off(ARRAYS); Ok(StoredCsr { version, fragments, fields, - outgoing: decode_direction(&batch, "out").ok_or_else(|| bad("outgoing arrays"))?, - incoming: decode_direction(&batch, "in").ok_or_else(|| bad("incoming arrays"))?, + outgoing: decode_direction(arrays).ok_or_else(|| bad("outgoing arrays"))?, + incoming: decode_direction(incoming).ok_or_else(|| bad("incoming arrays"))?, }) } -fn decode_direction(batch: &RecordBatch, prefix: &str) -> Option { - let column = |name: &str| batch.column_by_name(&format!("{prefix}_{name}")); - let sources = column("source")?.as_run_opt::()?; - let ends = sources.run_ends(); - let keys = sources.values().as_primitive_opt::()?; - if sources.offset() != 0 || keys.null_count() != 0 || keys.len() != ends.values().len() { +fn decode_direction(arrays: Vec>) -> Option { + let [keys, offsets, neighbors, rb, rm, rw]: [ScalarBuffer; ARRAYS] = + arrays.try_into().ok()?; + let edge_rows = PackedU64::from_parts(neighbors.len(), [rb, rm, rw])?; + if keys.len() + 1 != offsets.len() { return None; } - let mut offsets = Vec::with_capacity(keys.len() + 1); - offsets.push(0); - offsets.extend(ends.values().iter().map(|&end| end as u64)); - let u64s = |name: &str| { - let array = column(name)?.as_primitive_opt::()?; - (array.null_count() == 0).then(|| array.values().clone()) - }; - let csr = CsrIndex::from_parts(offsets.into(), u64s("neighbor")?, u64s("edge_row")?)?; - Adjacency::from_base(keys.values(), csr) + let csr = CsrIndex::from_packed(offsets, neighbors, edge_rows)?; + let keys: Vec = keys.iter().map(|&k| k as i64).collect(); + Adjacency::from_base(&keys, csr) } fn from_json(text: &str) -> Result { diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs index 913db2f..51d9089 100644 --- a/crates/lance-csr/src/engine.rs +++ b/crates/lance-csr/src/engine.rs @@ -857,6 +857,8 @@ async fn build_csr_entries( } } + // The one full build: the first, a refresh whose delta would pass the threshold, + // and optimize_csr() (`fold_deltas`) all end here. let started = std::time::Instant::now(); // Two projected scans of the endpoint columns, one per direction. let scan = || source.edge_stream(src.name(), dst.name()); diff --git a/docs/csr.md b/docs/csr.md index bc12cd0..af8affe 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -90,9 +90,11 @@ Node index !!! info "Memory cost" Expect about **95 bytes per edge** of extra peak memory while the CSR is built, for both directions - together (measured at 10M edges with `python/benchmarks/csr_expand.py`). About 32 bytes per edge stay - resident afterwards: each direction holds the neighbour key and the edge's row for every edge. The index - lives in memory even when the tables are read lazily from Lance. + together (measured at 10M edges with `python/benchmarks/csr_expand.py`). Afterwards, each direction keeps + 8 bytes per edge for the neighbour key and about 2 for the edge's row, which is stored + [packed](internals/indexes.md#packed-edge-rows), plus about 37 bytes per source node for its offset and its + entry in the map from key to position. On the GDC SF1 graph that's 732 MiB for 17.3 million edges, both + directions. The index lives in memory even when the tables are read lazily from Lance. During query execution, each CSR expansion partition keeps one input batch and collects at most one output chunk of neighbour positions before fetching properties. It yields that chunk before visiting @@ -394,14 +396,22 @@ print(stats["update"], stats["unindexed_edges"]) # rebuild 0 !!! tip "`refresh()` or `optimize_csr()`?" - Call `refresh()` whenever your queries should see new writes. It's cheap and incremental, and it already - rebuilds a CSR on its own once the delta passes 25% of the base. Call `optimize_csr()` only when - `csr_stats()` shows a delta built up below that threshold and you want lookups to stop carrying it: after a - bulk load, before a latency-sensitive run of queries, or on a schedule at a quiet time. It scans every edge - of the relationship dataset again, like the first build, so don't call it after every refresh. + They differ in which data they read and in when they rebuild: -`optimize_csr()` doesn't move to newer data, so call `refresh()` first. Like `refresh()`, it builds the -replacement in full before swapping it in, and the two run one at a time. + | | `refresh()` | `optimize_csr()` | + | --- | --- | --- | + | Data | Moves to the latest version of each dataset | Stays at the version the engine is pinned to | + | Usual effect on a CSR | Applies the new writes as a delta, reading only new fragments | Rebuilds it from a scan of all its edges | + | Rebuilds a CSR when | Its delta would pass 25% of the base | It carries any delta at all | + | Cost | In proportion to the new writes | Like the first build | + + So call `refresh()` whenever your queries should see new writes; it's cheap enough to run on a timer. Call + `optimize_csr()` after it when `csr_stats()` shows a delta below the threshold that you'd rather lookups + didn't carry: after a bulk load, before a latency-sensitive run of queries, or at a quiet time. When either + one rebuilds, it runs the same full build and gives the same CSR. + +Like `refresh()`, `optimize_csr()` builds the replacement in full before swapping it in, and the two run one at a +time. ## Storing the CSR @@ -454,19 +464,19 @@ print(open_engine().delete_stored_csr()) # 1 !!! note "Cost" - The file holds the base CSR's arrays, compressed with zstd: on SF1 about 9 bytes per edge, against the - 32 bytes per edge (plus 16 per source key in each direction) the CSR takes in memory. Node indexes are still - built at start-up. On the GDC SF1 graph read lazily (17.3 million edges over 23 relationship types), measured - on an Apple M5: + The file holds the CSR's arrays as they sit in memory, compressed with zstd: on SF1 about 8 bytes per edge, + 134 MiB in all. A load decompresses them and rebuilds only the map from key to position; node indexes are + still built at start-up. On the GDC SF1 graph read lazily (17.3 million edges over 23 relationship types), + measured on an Apple M5: | | Engine start | Peak memory | | --- | ---: | ---: | - | Without `persist_csr` | 1.55 s | 1.5 to 1.7 GiB | - | First start, building and storing 150 MiB | 2.9 s | 1.5 to 1.7 GiB | - | Later starts, loading | 0.58 s | 1.25 to 1.3 GiB | + | Without `persist_csr` | 1.7 s | 1.4 to 1.45 GiB | + | First start, building and storing 134 MiB | 2.6 s | 1.3 to 1.5 GiB | + | Later starts, loading | 0.46 s | 1.1 GiB | - Compressing adds about a second to the first start, and decompressing costs a load about 0.4 s. Query times - don't change. `python/benchmarks/csr_persist.py` measures the same cases on a synthetic graph, including a load + Compressing adds about 0.9 s to the first start; decompression takes most of a load. Query times don't + change. `python/benchmarks/csr_persist.py` measures the same cases on a synthetic graph, including a load with a delta. ## Should I turn it on? diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md index 3b7c92f..225bc09 100644 --- a/docs/internals/indexes.md +++ b/docs/internals/indexes.md @@ -39,9 +39,24 @@ each type has an outgoing and an incoming one. It is built from the relationship - each neighbour also records the row its edge came from: its Lance row address (fragment id in the high 32 bits, offset in the low 32), or its row number for in-memory input. -That makes 16 bytes per edge in each direction. The edge row is what lets a hop fetch relationship properties with -`take`, and what lets a refresh find deleted edges without reading the old dataset version. Edges with a null -endpoint are skipped: they could never satisfy the equality join the CSR replaces. +The edge row is what lets a hop fetch relationship properties with `take`, and what lets a refresh find deleted edges +without reading the old dataset version. Edges with a null endpoint are skipped: they could never satisfy the equality +join the CSR replaces. + +### Packed edge rows + +Unpacked, the neighbour and the edge row take 16 bytes per edge in each direction. Edge rows are stored packed, by +`PackedU64`: in blocks of 128 values, a block keeps its minimum and a bit width, and its values as bit-packed offsets +from that minimum. Neighbouring slots mostly come from nearby rows of the same fragment, so a block needs few bits; on +GDC SF1 edge rows take about 2 bytes per edge in each direction instead of 8, which cut the CSR's arrays from 613 to +419 MiB. Any position stays directly readable, +which traversal needs to resume part-way through a list, and reading a range loads each block's header once. Plain +hops never unpack edge rows; only hops that name their relationship, and refresh deletes, do. + +Neighbours stay a plain `u64` array. Packing them the same way roughly halved them on SF1 (their keys are sparse: +Comment keys span 2.2 trillion for 2 million rows, so a single width for the whole array, as a `u32` offset, wouldn't +fit at all), but every hop reads neighbours, and lookup-heavy queries got slower: custom Q30, which makes one lookup per comment +into lists of one neighbour, by about 22%. Each lookup read a block base, a header and a word instead of one value. ## The node index @@ -100,16 +115,20 @@ full comparison. With `persist_csr=True` (see [Storing the CSR](../csr.md#storing-the-csr)), `csr_store.rs` writes the base of both adjacencies after every full build of a Lance relationship dataset, to `_lance_csr/.arrow` at the dataset's -root. It's one Arrow IPC file holding one record batch, one row per edge in CSR position order: +root. The file holds the CSR exactly as it sits in memory, so a load neither builds nor packs anything. It's an +Arrow IPC file with one `UInt64` column, `values`, and twelve record batches of different lengths, one array each, +six per direction (outgoing first): -| Column | Type | Holds | -| --- | --- | --- | -| `out_source`, `in_source` | run-end encoded `Int64` | Source keys; the run ends are the CSR offsets | -| `out_neighbor`, `in_neighbor` | `UInt64` | Raw target keys | -| `out_edge_row`, `in_edge_row` | `UInt64` | Edge row addresses | +| Batch | Holds | +| --- | --- | +| 1 | Source keys (`i64` bit-cast) in CSR position order | +| 2 | Offsets | +| 3 | Neighbours | +| 4 to 6 | Packed edge rows: block bases, block headers (start bit and width), bit-packed words | -The schema metadata records the format, the dataset version, the endpoint column names, their Lance field ids and -types, and the fragments the base was built from. The delta is never stored. +The arrays only make sense to `csr_store.rs`, which validates every one before use; reading the file with pyarrow gives +the raw parts, not neighbour lists. The schema metadata records the format, the dataset version, the endpoint column +names, their Lance field ids and types, and the fragments the base was built from. The delta is never stored. When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as a refresh diffs two versions (see [How a refresh diffs Lance versions](#how-a-refresh-diffs-lance-versions)), so the file's own version doesn't need to @@ -117,8 +136,6 @@ exist any more. An empty diff uses the file as it is; otherwise the changes beco threshold. Rebuilds after start-up (from `refresh()` or `optimize_csr()`) don't look for a file, since the stored base is the one the engine already holds, but they do store the new one. -The arrays are compressed with zstd, Arrow IPC's own buffer compression, which made the SF1 files 4.7 times smaller -(711 to 150 MiB). `CsrIndex` keeps its arrays as Arrow `ScalarBuffer`s, so a loaded file's arrays are decompressed -straight into the buffers the CSR uses; only the source-key map is rebuilt, one insert per source. Decompression is -single-threaded, about 0.4 s of a load at SF1. Writes go through the dataset's object store with `put_if_absent`, which +Every batch is compressed with zstd, Arrow IPC's own buffer compression, on top of the packing. A load decompresses +the arrays, checks them, and rebuilds only the source-key map, one insert per source. Writes go through the dataset's object store with `put_if_absent`, which publishes the whole file at once on local disk and object storage; the compressed file is encoded in memory first. diff --git a/docs/lance.md b/docs/lance.md index 7989c7c..e24527f 100644 --- a/docs/lance.md +++ b/docs/lance.md @@ -137,7 +137,7 @@ memory. Joins, sorts and aggregations may still buffer their inputs. | --- | --- | | Lance dataset rows | On disk; read per query, only the needed columns and rows | | PyArrow inputs | In memory for the engine's lifetime, sliced across cores without copying | -| CSR adjacency (`use_csr=True`) | In memory, about 70 bytes per edge for both directions | +| CSR adjacency (`use_csr=True`) | In memory, about 20 bytes per edge for both directions plus about 37 per source node; see [the CSR's memory cost](csr.md#turning-it-on) | | Node index (`use_csr=True`) | In memory, one entry per node row | | Join-key statistics | In memory, a few numbers per key column | | Query results | In memory with `execute()`; streamed with `execute_batches()` | diff --git a/python/README.md b/python/README.md index 8db3c86..39a051d 100644 --- a/python/README.md +++ b/python/README.md @@ -132,7 +132,7 @@ for batch in engine.execute_batches("MATCH (p:Person) RETURN p.id"): Experimental: `CypherEngine(config, datasets, use_csr=True)` answers single-hop patterns over integer ids with an in-memory CSR adjacency, and fetches target rows by position instead of joining. The CSR is rebuilt on `refresh()` when the relationship table changes, and the per-label id index when its node -table changes. Results are identical; peak memory grows by about 70 bytes per edge (both directions). +table changes. Results are identical; peak memory grows by about 95 bytes per edge while the CSR is built (both directions). Write Lance data with `pylance` 13.x: the engine is built against Lance 13 and older or newer majors may write files it cannot read. From 459c0664e6ae59383ea7985215c7219caaf1110c Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:10:46 -0500 Subject: [PATCH 3/9] docs(csr): correct the peak memory units in the storing cost table --- docs/csr.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/csr.md b/docs/csr.md index af8affe..da90d56 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -471,9 +471,9 @@ print(open_engine().delete_stored_csr()) # 1 | | Engine start | Peak memory | | --- | ---: | ---: | - | Without `persist_csr` | 1.7 s | 1.4 to 1.45 GiB | - | First start, building and storing 134 MiB | 2.6 s | 1.3 to 1.5 GiB | - | Later starts, loading | 0.46 s | 1.1 GiB | + | Without `persist_csr` | 1.7 s | 1.37 to 1.42 GiB | + | First start, building and storing 134 MiB | 2.6 s | 1.25 to 1.46 GiB | + | Later starts, loading | 0.46 s | 1.06 to 1.12 GiB | Compressing adds about 0.9 s to the first start; decompression takes most of a load. Query times don't change. `python/benchmarks/csr_persist.py` measures the same cases on a synthetic graph, including a load From 13b1e19e452c9112a4d3e44ad617b634c9bab23f Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:13:38 -0500 Subject: [PATCH 4/9] docs: trim the readme and link the benchmark repositories --- README.md | 264 +++++----------------------------------- docs/AGENTS.md | 2 +- docs/sync_benchmarks.py | 4 +- 3 files changed, 32 insertions(+), 238 deletions(-) diff --git a/README.md b/README.md index 6cf74df..99db039 100644 --- a/README.md +++ b/README.md @@ -36,51 +36,21 @@ lance-csr is built for single-machine execution. It aims to: - **Push the limits of DataFusion for graph traversal.** We want to find out how far DataFusion's execution engine can go in powering multi-hop graph traversal, an area with little open-source work so far. -### Out of scope: distributed execution - -Graph queries across multiple machines are **explicitly out of scope** for this repository, to keep it free of -complexity that works against the goals above. **Issues and pull requests that target distributed execution will -be closed as out of scope.** - -If your use case involves distributed graph query processing or graph algorithms, that is exactly the kind of -engine we're building at HDC Labs. [Reach out to us](https://hyperdimensionalcomputing.ai/contact/) to chat more. - -## Why a CSR - -[lance-graph](https://github.com/lance-format/lance-graph) runs each relationship hop as a join between the edge and node tables. A -multi-hop query repeats that join at every hop, building a hash table and materializing every intermediate path each time. -[Compressed sparse row (CSR)](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format)) -is the standard compact layout for a sparse matrix, and so for a graph's adjacency: an offsets array indexed by node -points into one array of neighbor ids, so each node's neighbors sit next to each other. Expanding a hop becomes an -array lookup per node instead of a join. lance-csr builds this index from the relationship tables and runs it inside -DataFusion as extension operators, falling back to a join for any hop the CSR can't express. +Distributed execution is **out of scope**, and issues and pull requests that target it will be closed. If your use +case needs distributed graph queries or graph algorithms, that's the kind of engine we're building at HDC Labs: +[reach out to us](https://hyperdimensionalcomputing.ai/contact/). ## What this fork adds -- **CSR traversal** (`use_csr=True`) for single hops and variable-length paths, directed or undirected, updated - incrementally when the underlying Lance datasets change. -- **A Lance-native engine** that keeps each table either on disk (read lazily from a Lance snapshot) or in memory, - and `refresh()`es to new dataset versions. -- **Smarter planning**: patterns start at their most selective node, and DataFusion gets column statistics to size - its joins. -- **Much wider Cypher**. The Graph Data Council (GDC), formerly known as the Linked Data Benchmark Council (LDBC), - maintains the official Interactive complex queries. lance-csr runs them (IC14 without `reduce()`, which - openCypher leaves out), with `OPTIONAL MATCH`, `EXISTS` and pattern predicates, chained `WITH`, `UNWIND`, - list and pattern comprehensions, maps, relationship alternation, `shortestPath`, `$parameters` and more, - following openCypher's - [standardisation scope](https://github.com/opencypher/openCypher/blob/master/docs/standardisation-scope.adoc). -- **Correctness fixes** to behavior inherited from [lance-graph](https://github.com/lance-format/lance-graph) (for example, - undirected patterns dropped incoming edges), each with a regression test. -- **Current dependencies** (Lance 13, DataFusion 54, Arrow 58, PyO3 0.28), with Delta Lake and Unity Catalog support - removed. - -## Repository layout - -- `crates/lance-csr` – the Cypher-capable query engine implemented in Rust -- `python/` – PyO3 bindings and Python packages: - - `lance_csr` – thin wrapper around the Rust query engine - -See [`docs/project_structure.md`](docs/project_structure.md) for the full workspace layout. +- **CSR traversal** (`use_csr=True`): relationship hops and variable-length paths become array lookups instead of + joins, updated incrementally as the Lance datasets change. See [the CSR index](https://lance-csr.hdclabs.ai/csr/). +- **A Lance-native engine** that reads each table lazily from a pinned Lance snapshot or holds it in memory, and + `refresh()`es to new versions. +- **Much wider Cypher**, enough to run the Graph Data Council's Interactive complex queries: `OPTIONAL MATCH`, + pattern predicates, chained `WITH`, `UNWIND`, list and pattern comprehensions, `shortestPath`, `$parameters` + and more. +- **Correctness fixes** to behaviour inherited from lance-graph, each with a regression test, and current + dependencies (Lance 13, DataFusion 54, Arrow 58). ## Install @@ -88,48 +58,11 @@ See [`docs/project_structure.md`](docs/project_structure.md) for the full worksp uv add lance-csr ``` -Prebuilt wheels cover Linux x86-64 and native Apple Silicon (CPython 3.12+); no Rust toolchain is needed. -See [`python/README.md`](python/README.md) for details and usage. - -To use the engine from Rust, add the crate with `cargo add lance-csr` and see -[`crates/lance-csr/README.md`](crates/lance-csr/README.md) and the -[Rust guide](https://lance-csr.hdclabs.ai/rust/). - -## Build from source - -You need: - -- Rust, through [rustup](https://rustup.rs): `rust-toolchain.toml` pins the version, and rustup installs it on the - first build. -- `protoc`, which Lance's build needs: `brew install protobuf` on macOS, or - `sudo apt-get install -y protobuf-compiler` on Debian and Ubuntu. -- Python 3.12 or newer, and [`uv`](https://docs.astral.sh/uv/). - -Rust crates, from the repository root: - -```bash -cargo test --workspace -cargo clippy --workspace --all-targets -- -D warnings -cargo fmt --all --check -``` - -Python package, from `python/`. `uv sync --locked` installs the exact dependency versions in `uv.lock`, -including the development tools (maturin, pytest, ruff, pyright), into `python/.venv`: +Prebuilt wheels cover Linux x86-64 and native Apple Silicon (CPython 3.12+); no Rust toolchain is needed. From +Rust, run `cargo add lance-csr` and see the [Rust guide](https://lance-csr.hdclabs.ai/rust/). To build from +source, see [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md). -```bash -cd python -uv sync --locked --no-install-project # locked dependencies and dev tools -uv run --no-sync maturin develop # debug build of the extension into .venv -uv run --no-sync pytest python/tests # run the tests -``` - -`--no-install-project` leaves the package itself to `maturin develop`, which makes a debug build that compiles -quickly; uv would otherwise build it in release mode. `--no-sync` stops `uv run` from re-syncing, which would remove -that build. Re-run `maturin develop` after changing Rust code. `make install`, `make test` and `make lint` wrap -these steps. For timing queries, build with `maturin develop --release`: release builds use thin LTO and take -several minutes. See [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md) for more. - -## Python example: Cypher query +## Example ```python import pyarrow as pa @@ -155,167 +88,28 @@ result = query.execute({"Person": people}) print(result.to_pydict()) # {'name': ['Bob', 'David'], 'age': [34, 42]} ``` -## Python example: Direct SQL query - -For data analytics workflows where you prefer standard SQL, use `SqlQuery` or `SqlEngine`. No `GraphConfig` is needed: - -```python -import pyarrow as pa -from lance_csr import SqlQuery, SqlEngine - -person = pa.table({ - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Carol"], - "age": [28, 34, 29], -}) - -# One-off query -result = SqlQuery( - "SELECT name, age FROM person WHERE age > 30" -).execute({"person": person}) -print(result.to_pydict()) # {'name': ['Bob'], 'age': [34]} - -# Multi-query with cached context -engine = SqlEngine({"person": person}) -r1 = engine.execute("SELECT COUNT(*) AS cnt FROM person") -r2 = engine.execute("SELECT name FROM person ORDER BY age DESC LIMIT 2") -``` +More in the [getting started guide](https://lance-csr.hdclabs.ai/getting-started/) and the +[examples](examples/README.md). For plain SQL over the same datasets, see [SQL queries](https://lance-csr.hdclabs.ai/sql/). ## Benchmarks -The benchmark runners and raw results live in two sibling repositories, next to the other engines' runners, so -every number below can be reproduced from there. These are independent, unaudited measurements, not GDC Benchmark -Results. lance-csr runs with the CSR (`use_csr=True`) in two modes: *in memory* holds every table as Arrow in -memory, and *lazy* reads Lance datasets from disk as queries need them. Neo4j runs as a server in a Docker VM; the -other engines run embedded in the Python process. `docs/sync_benchmarks.py` regenerates the tables from the -repositories' committed results. - - - -### GDC SNB Interactive complex queries - -[GDC Interactive benchmark repository](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex) runs the -14 complex read queries (IC1–IC14) of the GDC SNB Interactive v1 workload over the SF1 dataset, with one fixed -parameter set per query and every result checked against a golden answer. lance-csr's -[runner](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex/tree/main/lance_csr) runs -Neo4j's Cypher text for IC1–IC13 unchanged; IC14 replaces `reduce()`, which openCypher leaves out, with pattern -comprehensions. - -Over the 14 queries, lance-csr's speed relative to each engine: - - - -| lance-csr mode | vs Neo4j | vs Kuzu | vs Ladybug | -| --- | ---: | ---: | ---: | -| In memory | 3.9× faster | 6.1× faster | 3.2× faster | -| Lazy (Lance on disk) | 1.8× faster | 2.8× faster | 1.3× faster | - - - -Mean query latency in milliseconds (lower is better, fastest per query in bold): - - - -| Query | lance-csr 0.3.1, in memory | lance-csr 0.3.1, lazy | Neo4j 2026.09.0 | Kuzu 0.11.3 | Ladybug 0.21.2 | -| --- | ---: | ---: | ---: | ---: | ---: | -| IC1 | 26.23 | 32.17 | 51.39 | **13.51** | n/a | -| IC2 | **2.74** | 26.18 | 44.73 | 24.97 | 15.56 | -| IC3 | **59.31** | 424.82 | 894.91 | 105.21 | 100.16 | -| IC4 | **5.94** | 13.44 | 13.05 | 29.66 | 11.83 | -| IC5 | **81.50** | 106.39 | 1,022.62 | 747.75 | 309.02 | -| IC6 | **24.68** | 27.14 | 231.25 | 1,114.76 | 234.21 | -| IC7 | 20.74 | 32.28 | **7.14** | 453.74 | 370.55 | -| IC8 | **2.04** | 3.06 | 3.00 | 73.44 | 4.95 | -| IC9 | **41.78** | 825.02 | 802.28 | 130.96 | 62.14 | -| IC10 | 100.21 | 136.60 | 91.68 | 74.90 | **13.69** | -| IC11 | 3.73 | 6.05 | 11.02 | 4.71 | **2.51** | -| IC12 | **19.77** | 26.75 | 133.87 | 92.26 | 133.84 | -| IC13 | 1.11 | 1.45 | **0.92** | 14.16 | 47.55 | -| IC14 | **46.80** | 47.88 | 522.56 | 1,871.19 | n/a | - - - - - -- Run on Apple M5, 24 GB, macOS 26.6.2, from the benchmark repository at commit [`f3e63fd`](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex/tree/f3e63fd2b7239a1364a864458f3e7ae07f0d833f). lance-csr was measured on 2026-10-27 and the other engines on 2026-10-24, on the same machine, with the repository's `run_benchmarks.sh` settings; see its README to reproduce the runs. -- Ratios are geometric means of the per-query ratios, over the queries both engines answered correctly. -- Ladybug 0.21.2: IC1 returned a wrong answer; IC14 failed (RuntimeError), so they are not timed. - - - - - -### GDC SNB SF1, 30-query workload - -[GDC SF1 benchmark repository](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc) runs 30 custom Cypher -queries over the GDC SNB SF1 dataset (3.18M nodes, 17.3M relationships), with every result checked against an -expected answer. lance-csr's [runner](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc/tree/main/lance_csr) -runs the same query text and Lance datasets as the [lance-graph](https://github.com/lance-format/lance-graph) runner. - -Over the 30 queries, lance-csr's speed relative to each engine: - - - -| lance-csr mode | vs Neo4j | vs Kuzu | vs Ladybug | vs lance-graph | -| --- | ---: | ---: | ---: | ---: | -| In memory | 3.6× faster | 4.1× faster | 2.6× faster | 4.6× faster | -| Lazy (Lance on disk) | 2.1× faster | 2.3× faster | 1.5× faster | 2.6× faster | - - - -Mean query latency in milliseconds (lower is better, fastest per query in bold): - - - -| Query | lance-csr 0.3.1, in memory | lance-csr 0.3.1, lazy | Neo4j 2026.09.0 | Kuzu 0.11.3 | Ladybug 0.21.2 | lance-graph 0.5.4 | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Q1 | **0.66** | 1.13 | 1.28 | 1.73 | 1.22 | 1.30 | -| Q2 | **0.51** | 1.33 | 3.26 | 1.32 | 1.12 | 2.34 | -| Q3 | **0.62** | 0.77 | 0.66 | 1.03 | 0.79 | 1.86 | -| Q4 | 0.48 | 0.75 | 2.66 | 0.85 | **0.39** | 2.98 | -| Q5 | **0.77** | 2.23 | 2.78 | 3.44 | 4.00 | 2.04 | -| Q6 | **0.41** | 0.71 | 2.60 | 0.66 | 0.79 | 0.73 | -| Q7 | 0.98 | 1.05 | **0.43** | 27.73 | 25.11 | 15.68 | -| Q8 | **0.61** | 1.75 | 7.29 | 2.72 | 3.38 | 1.14 | -| Q9 | 0.81 | 1.32 | **0.76** | 1.73 | 1.34 | 1.89 | -| Q10 | **0.70** | 0.91 | 3.79 | 1.55 | 1.78 | 26.69 | -| Q11 | **1.47** | 1.89 | 10.28 | 7.96 | 4.63 | 3.08 | -| Q12 | **0.69** | 6.80 | 2.55 | 17.16 | 7.84 | 18.68 | -| Q13 | **0.74** | 0.99 | 6.65 | 42.66 | 5.84 | 9.62 | -| Q14 | 0.48 | 0.65 | **0.46** | 1.56 | 0.90 | 2.91 | -| Q15 | **0.51** | 0.70 | 2.37 | 2.38 | 1.16 | 3.02 | -| Q16 | **0.62** | 1.06 | 1.26 | 1.78 | 1.04 | 4.33 | -| Q17 | **0.87** | 1.37 | 3.09 | 2.62 | 1.74 | 2.30 | -| Q18 | **0.63** | 1.07 | 3.11 | 1.51 | 1.22 | 1.76 | -| Q19 | **0.90** | 1.33 | 4.86 | 11.66 | 3.43 | 19.93 | -| Q20 | **2.05** | 16.45 | 421.92 | 13.25 | 14.15 | 3.13 | -| Q21 | 0.70 | 0.79 | 0.51 | **0.44** | 0.50 | 1.46 | -| Q22 | **0.65** | 0.88 | 2.66 | 23.34 | 7.46 | 13.41 | -| Q23 | **0.66** | 1.18 | 3.13 | 1.19 | 0.79 | 2.42 | -| Q24 | **0.63** | 1.00 | 0.72 | 1.23 | 0.81 | 1.75 | -| Q25 | **0.72** | 1.13 | 2.86 | 1.45 | 1.27 | 1.47 | -| Q26 | 0.67 | 1.02 | **0.65** | 3.37 | 1.15 | 2.93 | -| Q27 | **0.80** | 1.24 | 2.22 | 17.96 | 3.71 | 24.84 | -| Q28 | **1.36** | 2.25 | 2.37 | 1.51 | 1.78 | 2.50 | -| Q29 | **0.62** | 0.97 | 2.62 | 1.02 | 0.72 | 2.96 | -| Q30 | 43.67 | 47.50 | 1,359.98 | 146.52 | 97.77 | **34.31** | - - - - +lance-csr is measured against other graph engines on the Graph Data Council's SNB SF1 dataset (3.18M nodes, 17.3M +relationships). The runners, queries, expected answers and results live in two repositories, whose READMEs explain +how to reproduce the runs: -- Run on Apple M5, 24 GB, macOS 26.6.2, from the benchmark repository at commit [`b649bc1`](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc/tree/b649bc166cb5a04a542882fc070d9c4d83507c65). lance-csr was measured on 2026-10-27 and the other engines on 2026-10-24, on the same machine, with the repository's `run_benchmarks.sh` settings; see its README to reproduce the runs. -- Ratios are geometric means of the per-query ratios, over the queries both engines answered correctly. +- [graph-benchmark-ldbc-complex](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc-complex): the + 14 SNB Interactive complex read queries (IC1 to IC14). +- [graph-benchmark-ldbc](https://github.com/hyperdimensionalcomputingai/graph-benchmark-ldbc): 30 custom Cypher + queries. - +These are independent, unaudited measurements, not GDC Benchmark Results. ## Contributing Contributions within the [project's goals](#goals) are welcome, from people and from their coding agents. Every -pull request should explain the problem it solves and why the change is needed, and back that up with evidence: -a regression test for a bug fix, and before-and-after results from both benchmark suites for a code change. -[`CONTRIBUTING.md`](CONTRIBUTING.md) explains what to include and how to build the benchmark harness that -measures your branch against `main`. If you're unsure whether an idea fits, open an issue to discuss it first. +pull request should explain the problem it solves and back it up with evidence: a regression test for a bug fix, +and results from both benchmark suites for a code change. [`CONTRIBUTING.md`](CONTRIBUTING.md) explains what to +include and how to measure your branch against `main`. If you're unsure whether an idea fits, open an issue first. ## Credits diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 3da48c0..b295d0e 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -31,7 +31,7 @@ them. Build from the repository root with the locked build command above and publish `site/`. - Files that aren't pages (this file, `check_snippets.py`, `sync_benchmarks.py`, the uv project, `overrides/`, the brand README and previews) are listed under `exclude_docs` in `zensical.toml`. Add new non-page files there. -- Benchmark tables between `` markers in `benchmarks.md` and the root `README.md` are +- Benchmark tables between `` markers in `benchmarks.md` are generated; never edit them by hand. The runners and results live in graph-benchmark-ldbc-complex and graph-benchmark-ldbc. To publish a new run, merge it there, pin its commit in `sync_benchmarks.py`, and run `uv run --project docs --locked python docs/sync_benchmarks.py`. diff --git a/docs/sync_benchmarks.py b/docs/sync_benchmarks.py index 3ba1e94..5d4717c 100644 --- a/docs/sync_benchmarks.py +++ b/docs/sync_benchmarks.py @@ -1,4 +1,4 @@ -"""Rewrite the benchmark tables in README.md and docs/benchmarks.md from the benchmark repositories. +"""Rewrite the benchmark tables in docs/benchmarks.md from the benchmark repositories. Each benchmark repository commits pytest-benchmark's output for every engine to `results/`, with the machine and date in `results/environment.yml`. This script downloads those files at the @@ -25,7 +25,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -TARGETS = [ROOT / "README.md", ROOT / "docs" / "benchmarks.md"] +TARGETS = [ROOT / "docs" / "benchmarks.md"] # Columns: (results file stem, column title). The first two are lance-csr's modes; the rest are # the systems it is compared with. From 5720cb9e6e6e8a9be10728c42402938bcc1a9469 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:17:29 -0500 Subject: [PATCH 5/9] docs(csr): name the optimize_csr() section after what it does --- crates/lance-csr/src/engine.rs | 20 ++++++++++---------- docs/csr.md | 16 ++++++++-------- docs/internals/indexes.md | 2 +- python/python/lance_csr/cypher.py | 4 ++-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs index 51d9089..5450bee 100644 --- a/crates/lance-csr/src/engine.rs +++ b/crates/lance-csr/src/engine.rs @@ -281,7 +281,7 @@ struct CsrEntry { /// Rebuild instead of applying a delta once pending changes (appended edges + /// tombstones, accumulated since the last full build) would exceed this fraction -/// of the base. Lookups pay for the delta on every query, and a rebuild folds it in. +/// of the base. Lookups read the delta on every query, and a rebuild clears it. /// /// This threshold also bounds refresh cost: each delta refresh copies the whole /// accumulated delta (see the design note on `crate::csr_expand::Adjacency`). @@ -500,13 +500,13 @@ fn validate_sources(config: &GraphConfig, schemas: &HashMap) } /// Validate sources and register them in a fresh catalog and session context. -/// `fold_csr_deltas` rebuilds every CSR of `previous` that carries a delta. +/// `rebuild_deltas` rebuilds every CSR of `previous` that carries a delta. async fn build_engine_state( config: &GraphConfig, sources: HashMap, options: &EngineOptions, previous: Option<&EngineState>, - fold_csr_deltas: bool, + rebuild_deltas: bool, ) -> Result { let mut session_config = SessionConfig::new(); if let Some(partitions) = options.target_partitions { @@ -554,7 +554,7 @@ async fn build_engine_state( &sources, &schemas, previous, - fold_csr_deltas, + rebuild_deltas, options.persist_csr, ) .await?, @@ -806,7 +806,7 @@ async fn build_csr_entries( sources: &HashMap, schemas: &HashMap, previous: Option<&EngineState>, - fold_deltas: bool, + rebuild_deltas: bool, persist: bool, ) -> Result> { let mut entries = HashMap::new(); @@ -820,7 +820,7 @@ async fn build_csr_entries( }; if let Some(old) = previous.and_then(|p| p.csr.get(rel_type)) { let delta = old.outgoing.delta_edges() + old.outgoing.tombstones() > 0; - if old.source.same_data(source) && !(fold_deltas && delta) { + if old.source.same_data(source) && !(rebuild_deltas && delta) { entries.insert(rel_type.clone(), old.clone()); continue; } @@ -843,7 +843,7 @@ async fn build_csr_entries( } let previous_entry = previous.and_then(|p| p.csr.get(rel_type)); - if let Some(old) = previous_entry.filter(|_| !fold_deltas) { + if let Some(old) = previous_entry.filter(|_| !rebuild_deltas) { if let Some(entry) = try_apply_csr_delta(old, source, src.name(), dst.name()).await { entries.insert(rel_type.clone(), entry); continue; @@ -858,7 +858,7 @@ async fn build_csr_entries( } // The one full build: the first, a refresh whose delta would pass the threshold, - // and optimize_csr() (`fold_deltas`) all end here. + // and optimize_csr() (`rebuild_deltas`) all end here. let started = std::time::Instant::now(); // Two projected scans of the endpoint columns, one per direction. let scan = || source.edge_stream(src.name(), dst.name()); @@ -1153,7 +1153,7 @@ impl CypherEngine { Ok(()) } - /// Fold each CSR's delta into a new base, at the snapshot the engine is pinned to: the + /// Rebuild each CSR that carries a delta, at the snapshot the engine is pinned to: the /// CSR's counterpart of Lance's `optimize_indices()`. /// /// `refresh()` applies changes as a delta over the base CSR and rebuilds on its own @@ -1621,7 +1621,7 @@ mod tests { assert_eq!((rows == expected, stats.update), (true, "build")); assert_eq!(stored_files(&uri), ["3.arrow"], "older files are pruned"); - // optimize_csr() stores the folded base of a loaded delta. + // optimize_csr() stores the rebuilt base of a CSR loaded with a delta. write_knows(&uri, &[(5, 6)], WriteMode::Append).await; let config = GraphConfig::builder() .node_table("Person", "id") diff --git a/docs/csr.md b/docs/csr.md index da90d56..8695aa2 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -273,7 +273,7 @@ print(engine.csr_stats()["KNOWS"]) | `delta_edges` | Appended edges held outside the base CSR since its last full build. | | `tombstones` | Removed edges masked out since the last full build. | | `unindexed_edges` | Live edges outside the base CSR: appended since its last full build and not since removed. Lance index statistics call this `num_unindexed_rows`. | -| `update` | How the last build, `refresh()` or [`optimize_csr()`](#folding-the-delta-in) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). | +| `update` | How the last build, `refresh()` or [`optimize_csr()`](#rebuilding-the-csr) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). | | `version` | The relationship dataset's Lance version, or `None` for PyArrow input. | | `base_version` | The version of the last full build, or `None` for PyArrow input. It trails `version` while deltas apply. | | `build_ms` | Wall time of that build or delta, both directions. | @@ -308,15 +308,15 @@ flowchart TD Deleted and rewritten rows are matched by the row each CSR entry records, so only added fragments are read. Compaction and `cleanup_old_versions()` don't force a rebuild: nothing is read from the old version. - Once the accumulated delta would exceed **25%** of the base, or the dataset was overwritten, the CSR is - **rebuilt** and the delta folded in. + **rebuilt** from scratch, which clears the delta. - **Node-table writes never touch the CSR**: it stores raw target keys, not row positions. A changed node table rebuilds only its node index. - The replacement is validated in full before it's swapped in. If anything fails, nothing changes and the previous snapshot stays usable. Queries already running finish on the snapshot they started with. The 25% threshold is fixed: lookups pay a little for every pending delta edge, and a rebuild clears them. -`python/benchmarks/csr_delta_accumulated.py` measures that trade-off. To fold the delta in sooner, call -[`optimize_csr()`](#folding-the-delta-in). +`python/benchmarks/csr_delta_accumulated.py` measures that trade-off. To rebuild sooner, call +[`optimize_csr()`](#rebuilding-the-csr). ### How to keep the CSR fresh @@ -329,7 +329,7 @@ The engine doesn't watch your datasets for changes. Keeping the CSR in step with 3. **Check what it did with [`csr_stats()`](#csr_stats)**: `update` says whether the CSR was `reused`, updated with a `delta` or fully rebuilt (`rebuild`). `base_version` and `unindexed_edges` say how far the base CSR trails the data. Answers are exact either way: delta edges are part of every lookup. -4. **Optionally, fold the delta in with [`optimize_csr()`](#folding-the-delta-in)** at a quiet moment, so +4. **Optionally, rebuild with [`optimize_csr()`](#rebuilding-the-csr)** at a quiet moment, so lookups stop carrying it. === "Python" @@ -368,7 +368,7 @@ than you clean up old versions to keep updates incremental. In-memory PyArrow tables never change. To query new in-memory data, build a new engine. -### Folding the delta in +### Rebuilding the CSR with `optimize_csr()` { #rebuilding-the-csr } The CSR follows Lance's own index design. A Lance index covers the fragments it was built from, and data written after that stays unindexed: searches combine indexed results with a scan of the unindexed rows, so @@ -389,7 +389,7 @@ up with `version` and `update` says `rebuild`. A CSR without a delta is kept as ```python engine.refresh() # applies the new edges as a delta -engine.optimize_csr() # folds it into a new base +engine.optimize_csr() # rebuilds the CSR, now including those edges stats = engine.csr_stats()["KNOWS"] print(stats["update"], stats["unindexed_edges"]) # rebuild 0 ``` @@ -445,7 +445,7 @@ files it recognises. [`refresh()`](#keeping-the-csr-current) applies them. `update` says `loaded` and `base_version` names the stored version. That holds after compaction and `cleanup_old_versions()` too. - **Past the 25% threshold**, the engine rebuilds the CSR and stores the new version. Every full build is stored, - including those from `refresh()` and [`optimize_csr()`](#folding-the-delta-in), and older files are deleted + including those from `refresh()` and [`optimize_csr()`](#rebuilding-the-csr), and older files are deleted once a newer one is written. - **A file that can't be used** (corrupt, written for other endpoint columns or by another format) is ignored and rebuilt, so a bad file costs a build, never a wrong answer. diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md index 225bc09..6d23cc8 100644 --- a/docs/internals/indexes.md +++ b/docs/internals/indexes.md @@ -108,7 +108,7 @@ for a full rebuild. is raised. The base and delta map onto Lance's own index design: the delta plays the part of Lance's unindexed rows, and -`optimize_csr()` the part of `optimize_indices()`. [Folding the delta in](../csr.md#folding-the-delta-in) has the +`optimize_csr()` the part of `optimize_indices()`. [Rebuilding the CSR with `optimize_csr()`](../csr.md#rebuilding-the-csr) has the full comparison. ## Stored CSR files diff --git a/python/python/lance_csr/cypher.py b/python/python/lance_csr/cypher.py index 1c8cefe..c8ce1ab 100644 --- a/python/python/lance_csr/cypher.py +++ b/python/python/lance_csr/cypher.py @@ -271,7 +271,7 @@ def refresh(self) -> None: self._inner.refresh() def optimize_csr(self) -> None: - """Fold each CSR's delta into a new base (experimental; needs `use_csr=True`). + """Rebuild each CSR that carries a delta (experimental; needs `use_csr=True`). The CSR's counterpart of Lance's `optimize_indices()`. [`refresh`][lance_csr.CypherEngine.refresh] applies Lance writes as a delta @@ -286,7 +286,7 @@ def optimize_csr(self) -> None: `"rebuild"`, `unindexed_edges` 0 and `base_version` equal to `version`. As for `refresh`, the replacement is validated in full before it's swapped in, and the two run one at a time. See - [Folding the delta in](csr.md#folding-the-delta-in). + [Rebuilding the CSR](csr.md#rebuilding-the-csr). """ self._inner.optimize_csr() From b069120d6ebd57759981fc5de00e508fb9ee660c Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:20:53 -0500 Subject: [PATCH 6/9] docs(csr): lead with the two ways to keep the csr current --- docs/csr.md | 132 ++++++++++++++++++++++++---------------------------- 1 file changed, 62 insertions(+), 70 deletions(-) diff --git a/docs/csr.md b/docs/csr.md index 8695aa2..aa9d6d6 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -290,47 +290,26 @@ print(sorted(engine.node_index_stats())) # ['Person'] ## Keeping the CSR current -In-memory PyArrow tables never change, so their CSR never needs updating. For Lance datasets, the engine -is pinned to a version until you call [`refresh()`](lance.md#refresh), which updates the CSR incrementally: +An engine over Lance datasets is pinned to one version of each, so its CSR doesn't see later writes until you ask. +Two calls keep it current: -```mermaid -flowchart TD - R["engine.refresh()"] --> V{"Relationship dataset
changed?"} - V -- no --> U["reused"] - V -- yes --> D{"Same endpoint
columns?"} - D -- no --> RB["rebuild"] - D -- yes --> S{"Pending changes > 25%
of the base CSR?"} - S -- yes --> RB - S -- no --> DL["delta: appended edges +
tombstones for deleted ones"] -``` +1. **[`refresh()`](#refresh)** moves the engine to the latest data and applies new edges as a delta on top of the + CSR. Call it whenever your queries should see new writes. +2. **[`optimize_csr()`](#rebuilding-the-csr)** rebuilds a CSR that carries a delta, so lookups stop reading it. + Call it now and then, once a delta has built up. + +In-memory PyArrow tables never change, so their CSR needs neither. To query new in-memory data, build a new engine. + +### `refresh()`: pick up new writes { #refresh } -- **Appends** become delta edges and **deletes** become tombstones, both applied on top of the base CSR. - Deleted and rewritten rows are matched by the row each CSR entry records, so only added fragments are - read. Compaction and `cleanup_old_versions()` don't force a rebuild: nothing is read from the old version. -- Once the accumulated delta would exceed **25%** of the base, or the dataset was overwritten, the CSR is - **rebuilt** from scratch, which clears the delta. -- **Node-table writes never touch the CSR**: it stores raw target keys, not row positions. A changed node - table rebuilds only its node index. -- The replacement is validated in full before it's swapped in. If anything fails, nothing changes and the - previous snapshot stays usable. Queries already running finish on the snapshot they started with. - -The 25% threshold is fixed: lookups pay a little for every pending delta edge, and a rebuild clears them. -`python/benchmarks/csr_delta_accumulated.py` measures that trade-off. To rebuild sooner, call -[`optimize_csr()`](#rebuilding-the-csr). - -### How to keep the CSR fresh - -The engine doesn't watch your datasets for changes. Keeping the CSR in step with the data is up to you: - -1. **Write with any Lance writer**: `pylance`, LanceDB or another process. -2. **Call `refresh()` after the writes your queries should see.** When a writer updates several tables, call it - after all of them commit (see [snapshots are per dataset](lance.md#lazy-version-pinned-reads)). A refresh - with nothing new reuses the existing CSR, so calling it on a timer or before each batch of queries is fine. -3. **Check what it did with [`csr_stats()`](#csr_stats)**: `update` says whether the CSR was `reused`, updated - with a `delta` or fully rebuilt (`rebuild`). `base_version` and `unindexed_edges` say how far the base - CSR trails the data. Answers are exact either way: delta edges are part of every lookup. -4. **Optionally, rebuild with [`optimize_csr()`](#rebuilding-the-csr)** at a quiet moment, so - lookups stop carrying it. +The engine doesn't watch your Lance datasets. After writes your queries should see, from `pylance`, LanceDB or another +process, call `refresh()`; when a writer updates several tables, call it once all of them have committed (see +[snapshots are per Lance dataset](lance.md#lazy-version-pinned-reads)). A refresh with nothing new reuses everything, so +running it on a timer or before each batch of queries is fine. + +It exists so that new writes don't cost a full rebuild. Appended and deleted edges become a delta on top of the +existing CSR, and only the new fragments are read. [`csr_stats()`](#csr_stats) shows what it did: `update` is +`reused`, `delta` or `rebuild`. === "Python" @@ -361,31 +340,30 @@ The engine doesn't watch your datasets for changes. Keeping the CSR in step with --8<-- "crates/lance-csr/examples/embedded_query.rs:refresh" ``` -A refresh can only apply a delta when it can compare the version the engine is pinned to with the latest one. -If you overwrite a relationship dataset, or remove the pinned version's files (for example with -`cleanup_old_versions()`) before refreshing, the next refresh rebuilds that CSR in full. Refresh more often -than you clean up old versions to keep updates incremental. - -In-memory PyArrow tables never change. To query new in-memory data, build a new engine. +??? info "How `refresh()` decides" -### Rebuilding the CSR with `optimize_csr()` { #rebuilding-the-csr } + ```mermaid + flowchart LR + R["refresh()"] -- "no edge changes" --> U["reused"] + R -- "changes under 25% of the base" --> DL["delta"] + R -- "25% or more, or new endpoint columns" --> RB["rebuild"] + ``` -The CSR follows Lance's own index design. A Lance index covers the fragments it was built from, and data -written after that stays unindexed: searches combine indexed results with a scan of the unindexed rows, so -answers stay correct while latency grows, until you call `optimize_indices()` (see Lance's -[index maintenance guide](https://lance.org/guide/indexing/#maintain-indexes-after-appends) and the -[index format](https://lance.org/format/index/#basic-concepts)). The CSR maps onto that one to one: + - Deleted and rewritten rows are matched by the edge row each CSR entry records, so compaction and + `cleanup_old_versions()` don't force a rebuild: nothing is read from the old version. + - Past 25% of the base, or after the Lance dataset is overwritten, the CSR is rebuilt from scratch. The threshold is + fixed: every lookup reads the delta, and a rebuild clears it. `python/benchmarks/csr_delta_accumulated.py` + measures that trade-off. + - Node-table writes never touch the CSR, which stores raw target keys. Only that label's node index is rebuilt. + - The replacement is built in full before it's swapped in. If anything fails, nothing changes, and queries + already running finish on the snapshot they started with. -| Lance index | CSR | -| --- | --- | -| The fragments an index was built from | The base CSR, at `base_version` | -| Unindexed rows, scanned alongside the index | The delta (appended edges and tombstones), read by every lookup | -| `num_unindexed_rows` in index statistics | `unindexed_edges` in `csr_stats()` | -| `optimize_indices()` | [`optimize_csr()`][lance_csr.CypherEngine.optimize_csr] | +### `optimize_csr()`: rebuild to clear the delta { #rebuilding-the-csr } -`optimize_csr()` rebuilds every CSR that carries a delta, at the version the engine is pinned to, so the base -covers all the data again: `unindexed_edges`, `delta_edges` and `tombstones` drop to 0, `base_version` catches -up with `version` and `update` says `rebuild`. A CSR without a delta is kept as it is, and answers never change. +Every lookup reads the delta next to the base CSR, so a delta that stays below the 25% threshold slows lookups a +little for as long as it's there. `optimize_csr()` clears it when you choose: it rebuilds every CSR that carries a +delta, at the version the engine is pinned to, from a scan of all its edges. Use it after a bulk load, before a +latency-sensitive run of queries, or at a quiet time, not after every refresh. Answers never change. ```python engine.refresh() # applies the new edges as a delta @@ -394,24 +372,38 @@ stats = engine.csr_stats()["KNOWS"] print(stats["update"], stats["unindexed_edges"]) # rebuild 0 ``` -!!! tip "`refresh()` or `optimize_csr()`?" +Afterwards `delta_edges`, `tombstones` and `unindexed_edges` are 0 and `base_version` equals `version`. A CSR +without a delta is kept as it is. It doesn't move to newer data, so call `refresh()` first. + +??? info "How this maps onto Lance's indexes" + + A Lance index covers the fragments it was built from. Data written after that stays unindexed: searches combine + indexed results with a scan of the unindexed rows, so answers stay correct while latency grows, until you call + `optimize_indices()` (see Lance's + [index maintenance guide](https://lance.org/guide/indexing/#maintain-indexes-after-appends) and the + [index format](https://lance.org/format/index/#basic-concepts)). The CSR maps onto that one to one: + + | Lance index | CSR | + | --- | --- | + | The fragments an index was built from | The base CSR, at `base_version` | + | Unindexed rows, scanned alongside the index | The delta (appended edges and tombstones), read by every lookup | + | `num_unindexed_rows` in index statistics | `unindexed_edges` in `csr_stats()` | + | `optimize_indices()` | [`optimize_csr()`][lance_csr.CypherEngine.optimize_csr] | - They differ in which data they read and in when they rebuild: +### Which one to call + +!!! tip "`refresh()` or `optimize_csr()`?" | | `refresh()` | `optimize_csr()` | | --- | --- | --- | - | Data | Moves to the latest version of each dataset | Stays at the version the engine is pinned to | + | Data | Moves to the latest version of each Lance dataset | Stays at the version the engine is pinned to | | Usual effect on a CSR | Applies the new writes as a delta, reading only new fragments | Rebuilds it from a scan of all its edges | | Rebuilds a CSR when | Its delta would pass 25% of the base | It carries any delta at all | | Cost | In proportion to the new writes | Like the first build | + | When to call it | Whenever queries should see new writes | Now and then, after `refresh()`, once a delta has built up | - So call `refresh()` whenever your queries should see new writes; it's cheap enough to run on a timer. Call - `optimize_csr()` after it when `csr_stats()` shows a delta below the threshold that you'd rather lookups - didn't carry: after a bulk load, before a latency-sensitive run of queries, or at a quiet time. When either - one rebuilds, it runs the same full build and gives the same CSR. - -Like `refresh()`, `optimize_csr()` builds the replacement in full before swapping it in, and the two run one at a -time. + When either one rebuilds, it runs the same full build and gives the same CSR. Both build the replacement in + full before swapping it in, and they run one at a time. ## Storing the CSR From ccb009199da57a86da9e575a7e0336a2c8ea96f4 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 12:23:45 -0500 Subject: [PATCH 7/9] docs(csr): explain why the csr falls behind and what each call does --- docs/csr.md | 92 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/docs/csr.md b/docs/csr.md index aa9d6d6..74aa3dc 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -290,26 +290,40 @@ print(sorted(engine.node_index_stats())) # ['Person'] ## Keeping the CSR current -An engine over Lance datasets is pinned to one version of each, so its CSR doesn't see later writes until you ask. -Two calls keep it current: +A Lance dataset can change at any time. You, another process or LanceDB can append rows, delete them or compact +the files, and every write commits a new version of the dataset. The engine doesn't follow those changes as they +happen: when you create it, it opens each Lance dataset at its current version, builds the CSR from that version, +and keeps answering queries from that snapshot until you tell it to look again. That keeps queries consistent with +each other, but it also means the CSR falls behind as soon as anyone writes to the data. -1. **[`refresh()`](#refresh)** moves the engine to the latest data and applies new edges as a delta on top of the - CSR. Call it whenever your queries should see new writes. -2. **[`optimize_csr()`](#rebuilding-the-csr)** rebuilds a CSR that carries a delta, so lookups stop reading it. - Call it now and then, once a delta has built up. +Two calls bring it back up to date, and they do different jobs: -In-memory PyArrow tables never change, so their CSR needs neither. To query new in-memory data, build a new engine. +1. **[`refresh()`](#refresh)** moves the engine to the newest version of each Lance dataset. Instead of rebuilding + the CSR, it records the edges added or removed since the last build as a *delta*, a small set of changes kept + next to the CSR. Call it whenever your queries should see new writes. +2. **[`optimize_csr()`](#rebuilding-the-csr)** rebuilds a CSR from scratch, so it no longer needs its delta. Call + it occasionally, once enough changes have built up that you'd rather lookups stopped working through them. -### `refresh()`: pick up new writes { #refresh } +If you gave the engine in-memory PyArrow tables instead of Lance datasets, neither call applies: those tables can't +change underneath the engine, so their CSR is never out of date. To query new in-memory data, create a new engine. -The engine doesn't watch your Lance datasets. After writes your queries should see, from `pylance`, LanceDB or another -process, call `refresh()`; when a writer updates several tables, call it once all of them have committed (see -[snapshots are per Lance dataset](lance.md#lazy-version-pinned-reads)). A refresh with nothing new reuses everything, so -running it on a timer or before each batch of queries is fine. +### `refresh()`: pick up new writes { #refresh } -It exists so that new writes don't cost a full rebuild. Appended and deleted edges become a delta on top of the -existing CSR, and only the new fragments are read. [`csr_stats()`](#csr_stats) shows what it did: `update` is -`reused`, `delta` or `rebuild`. +Because each Lance dataset can evolve independently as you write to it, the engine has to be told that a newer +version is available. `refresh()` does that: it checks every Lance dataset for a newer version and moves the engine to +it. Call it after the writes your queries should see, whether they came from `pylance`, LanceDB or another process. +If one writer updates several tables, wait until all of them have committed before you refresh, since each Lance +dataset is pinned on its own (see [snapshots are per Lance dataset](lance.md#lazy-version-pinned-reads)). When nothing +has changed, a refresh keeps what the engine already has and returns quickly, so calling it on a timer or before +each batch of queries is fine. + +Rebuilding the whole CSR on every refresh would mean scanning every edge again, even when only a handful changed. +So `refresh()` reads only the new parts of each relationship dataset and applies the changes on top of the CSR from +the last full build, which the rest of this page calls the *base*. Appended edges go into the delta, and deleted +edges are marked so that lookups skip them; `csr_stats()` counts those marks as `tombstones`. Answers are exactly +what a full rebuild would give. To see what a refresh did, check `update` in [`csr_stats()`](#csr_stats): `reused` +means nothing changed, `delta` means the changes were applied, and `rebuild` means the CSR was built again from +scratch, which happens in the cases below. === "Python" @@ -349,21 +363,26 @@ existing CSR, and only the new fragments are read. [`csr_stats()`](#csr_stats) s R -- "25% or more, or new endpoint columns" --> RB["rebuild"] ``` - - Deleted and rewritten rows are matched by the edge row each CSR entry records, so compaction and - `cleanup_old_versions()` don't force a rebuild: nothing is read from the old version. - - Past 25% of the base, or after the Lance dataset is overwritten, the CSR is rebuilt from scratch. The threshold is - fixed: every lookup reads the delta, and a rebuild clears it. `python/benchmarks/csr_delta_accumulated.py` - measures that trade-off. - - Node-table writes never touch the CSR, which stores raw target keys. Only that label's node index is rebuilt. - - The replacement is built in full before it's swapped in. If anything fails, nothing changes, and queries - already running finish on the snapshot they started with. + - A refresh rebuilds a CSR instead of adding to its delta once the changes since the last full build would reach + 25% of the base, or when the relationship dataset was overwritten or its endpoint columns changed. The + threshold is fixed: every lookup reads the delta as well as the base, so a large delta slows lookups down, and + a rebuild clears it. `python/benchmarks/csr_delta_accumulated.py` measures that trade-off. + - Deleted and rewritten rows are recognised by the edge row that each CSR entry records, so compacting a dataset + or running `cleanup_old_versions()` doesn't force a rebuild: nothing needs to be read from the old version. + - Writes to a node table never touch the CSR, because the CSR stores the target nodes' keys rather than their row + positions. Only that label's node index is rebuilt. + - The new CSR is built in full before the engine switches to it. If anything fails along the way, nothing + changes, and queries that were already running finish on the snapshot they started with. ### `optimize_csr()`: rebuild to clear the delta { #rebuilding-the-csr } -Every lookup reads the delta next to the base CSR, so a delta that stays below the 25% threshold slows lookups a -little for as long as it's there. `optimize_csr()` clears it when you choose: it rebuilds every CSR that carries a -delta, at the version the engine is pinned to, from a scan of all its edges. Use it after a bulk load, before a -latency-sensitive run of queries, or at a quiet time, not after every refresh. Answers never change. +The delta keeps refreshes cheap, but every lookup has to check it as well as the base. A refresh only rebuilds once +the delta reaches 25% of the base, so a smaller delta can stay around indefinitely and slow lookups a little the +whole time. `optimize_csr()` lets you clear it at a moment you choose. It rebuilds every CSR that has a delta, from a +scan of all its edges, at the version the engine is already pinned to, and leaves CSRs without a delta alone. That +costs about as much as building the CSR the first time, so it's best kept for after a bulk load, before a run of +latency-sensitive queries, or a quiet period, rather than called after every refresh. Query results are the same +before and after. ```python engine.refresh() # applies the new edges as a delta @@ -372,14 +391,16 @@ stats = engine.csr_stats()["KNOWS"] print(stats["update"], stats["unindexed_edges"]) # rebuild 0 ``` -Afterwards `delta_edges`, `tombstones` and `unindexed_edges` are 0 and `base_version` equals `version`. A CSR -without a delta is kept as it is. It doesn't move to newer data, so call `refresh()` first. +Afterwards, `csr_stats()` shows `delta_edges`, `tombstones` and `unindexed_edges` at 0, and `base_version` matches +`version`. `optimize_csr()` doesn't look for newer data, so call `refresh()` first if the rebuild should include the +latest writes. ??? info "How this maps onto Lance's indexes" - A Lance index covers the fragments it was built from. Data written after that stays unindexed: searches combine - indexed results with a scan of the unindexed rows, so answers stay correct while latency grows, until you call - `optimize_indices()` (see Lance's + The CSR is maintained the same way Lance maintains its own indexes. A Lance index covers the fragments it was + built from, and data written after that stays unindexed: searches combine the index's results with a scan of the + unindexed rows, so answers stay correct while latency slowly grows, until you call `optimize_indices()` (see + Lance's [index maintenance guide](https://lance.org/guide/indexing/#maintain-indexes-after-appends) and the [index format](https://lance.org/format/index/#basic-concepts)). The CSR maps onto that one to one: @@ -394,6 +415,9 @@ without a delta is kept as it is. It doesn't move to newer data, so call `refres !!! tip "`refresh()` or `optimize_csr()`?" + In short, `refresh()` keeps the engine up to date at little cost, and `optimize_csr()` now and then clears the + delta that refreshes leave behind. + | | `refresh()` | `optimize_csr()` | | --- | --- | --- | | Data | Moves to the latest version of each Lance dataset | Stays at the version the engine is pinned to | @@ -402,8 +426,8 @@ without a delta is kept as it is. It doesn't move to newer data, so call `refres | Cost | In proportion to the new writes | Like the first build | | When to call it | Whenever queries should see new writes | Now and then, after `refresh()`, once a delta has built up | - When either one rebuilds, it runs the same full build and gives the same CSR. Both build the replacement in - full before swapping it in, and they run one at a time. + Whenever either call rebuilds a CSR, it runs the same full build and ends up with the same CSR. Both build the + replacement completely before the engine switches to it, and they never run at the same time. ## Storing the CSR From 36200dfb8ca8f4289f9d4a0ad5258769ebb32661 Mon Sep 17 00:00:00 2001 From: Prashanth Rao <35005448+prrao87@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:49:50 -0500 Subject: [PATCH 8/9] refactor(engine): rename refresh() to checkout_latest() to match lance (#110) * refactor(engine): rename refresh() to checkout_latest() to match lance Lance moves a dataset to its newest version with checkout_latest() and has no refresh(). CypherEngine.refresh() does the same for every dataset the engine reads, so it is now checkout_latest(), in Rust and Python, which also pairs it with optimize_csr() the way Lance pairs checkout_latest() with optimize_indices(). refresh() stays as a deprecated alias: Python warns with DeprecationWarning, and Rust marks it #[deprecated]. Docs, examples, tests and benchmarks use the new name, and the lance_snapshot_refresh.py example is now lance_snapshots.py. * docs: say checkout_latest() follows lance and optimize_csr() rebuilds the csr --- README.md | 2 +- crates/lance-csr-python/src/graph.rs | 9 +- crates/lance-csr/README.md | 6 +- crates/lance-csr/examples/embedded_query.rs | 12 +-- crates/lance-csr/src/engine.rs | 78 ++++++++------- crates/lance-csr/tests/test_engine.rs | 8 +- docs/AGENTS.md | 2 +- docs/benchmarks.md | 6 +- docs/csr.md | 61 ++++++------ docs/cypher.md | 10 +- docs/execution.md | 8 +- docs/getting-started.md | 2 +- docs/index.md | 2 +- docs/internals/index.md | 2 +- docs/internals/indexes.md | 6 +- docs/internals/optimizer.md | 2 +- docs/lance.md | 51 ++++++---- docs/rust.md | 10 +- docs/sql.md | 2 +- examples/README.md | 6 +- ...snapshot_refresh.py => lance_snapshots.py} | 14 +-- python/README.md | 12 +-- python/benchmarks/csr_delta.py | 6 +- python/benchmarks/csr_delta_accumulated.py | 4 +- python/benchmarks/csr_expand.py | 4 +- python/benchmarks/engine_cache.py | 10 +- python/python/lance_csr/_internal/graph.pyi | 1 + python/python/lance_csr/cypher.py | 96 +++++++++++-------- python/python/tests/test_call_subqueries.py | 2 +- python/python/tests/test_engine_csr.py | 10 +- python/python/tests/test_engine_csr_delta.py | 38 ++++---- python/python/tests/test_engine_lance.py | 33 ++++--- python/python/tests/test_path_values.py | 4 +- python/python/tests/test_target_partitions.py | 2 +- python/python/tests/test_typing.py | 2 +- python/python/tests/typing/public_api.py | 2 +- 36 files changed, 292 insertions(+), 233 deletions(-) rename examples/{lance_snapshot_refresh.py => lance_snapshots.py} (84%) diff --git a/README.md b/README.md index 99db039..bd64b03 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ case needs distributed graph queries or graph algorithms, that's the kind of eng - **CSR traversal** (`use_csr=True`): relationship hops and variable-length paths become array lookups instead of joins, updated incrementally as the Lance datasets change. See [the CSR index](https://lance-csr.hdclabs.ai/csr/). - **A Lance-native engine** that reads each table lazily from a pinned Lance snapshot or holds it in memory, and - `refresh()`es to new versions. + `checkout_latest()`es to new versions. - **Much wider Cypher**, enough to run the Graph Data Council's Interactive complex queries: `OPTIONAL MATCH`, pattern predicates, chained `WITH`, `UNWIND`, list and pattern comprehensions, `shortestPath`, `$parameters` and more. diff --git a/crates/lance-csr-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs index 0725dd3..d57be91 100644 --- a/crates/lance-csr-python/src/graph.rs +++ b/crates/lance-csr-python/src/graph.rs @@ -1119,11 +1119,16 @@ impl CypherEngine { .map_err(graph_error_to_pyerr) } - fn refresh(&self, py: Python) -> PyResult<()> { - RT.block_on(Some(py), self.inner.refresh())? + fn checkout_latest(&self, py: Python) -> PyResult<()> { + RT.block_on(Some(py), self.inner.checkout_latest())? .map_err(graph_error_to_pyerr) } + /// Deprecated alias; the Python wrapper warns before calling it. + fn refresh(&self, py: Python) -> PyResult<()> { + self.checkout_latest(py) + } + fn optimize_csr(&self, py: Python) -> PyResult<()> { RT.block_on(Some(py), self.inner.optimize_csr())? .map_err(graph_error_to_pyerr) diff --git a/crates/lance-csr/README.md b/crates/lance-csr/README.md index 1b4a9c4..167ddd8 100644 --- a/crates/lance-csr/README.md +++ b/crates/lance-csr/README.md @@ -15,7 +15,7 @@ crate. - Semantic validation with typed `GraphError` diagnostics - Pluggable execution strategies (DataFusion planner by default, Lance Native placeholder) - `CypherEngine`: register datasets once, optionally build the CSR, and serve many queries -- Lance datasets read lazily and pinned to a version until `refresh()` +- Lance datasets read lazily and pinned to a version until `checkout_latest()` - Async query execution that returns Arrow `RecordBatch` results or a stream - JSON-serializable parameter binding for reusable query templates - Logical plan debugging via `CypherQuery::explain` @@ -41,7 +41,7 @@ let batch = engine ``` `examples/embedded_query.rs` is a complete program that writes two Lance datasets, queries them, and picks up a -new write with `refresh()`: +new write with `checkout_latest()`: ```bash cargo run -p lance-csr --example embedded_query @@ -154,7 +154,7 @@ clauses, patterns, expressions and functions the engine runs. - `logical_plan` – Builders for graph logical plans. - `datafusion_planner` – DataFusion-based execution planning. - `config` – Graph configuration types and builders. -- `engine` – `CypherEngine`: cached snapshot, CSR adjacencies, `refresh()`. +- `engine` – `CypherEngine`: cached snapshot, CSR adjacencies, `checkout_latest()`. - `query` – High level `CypherQuery` API and runtime. - `csr_index`, `csr_expand` – The CSR adjacency index and the traversal operators that use it. - `error` – `GraphError` and result helpers. diff --git a/crates/lance-csr/examples/embedded_query.rs b/crates/lance-csr/examples/embedded_query.rs index 71cf107..5e7ecff 100644 --- a/crates/lance-csr/examples/embedded_query.rs +++ b/crates/lance-csr/examples/embedded_query.rs @@ -2,7 +2,7 @@ //! //! Writes a small social graph as two Lance datasets in a temporary directory, opens //! them in a CSR-backed engine, runs a two-hop query, appends an edge and picks it up -//! with `refresh()`, runs a query under a deadline, and repeats the in-memory Arrow +//! with `checkout_latest()`, runs a query under a deadline, and repeats the in-memory Arrow //! examples of the home page and the getting started guide. The documentation includes //! this file section by section (the `--8<--` markers), so keep it compiling and its printed //! output in step. @@ -80,7 +80,7 @@ async fn main() -> lance_csr::Result<()> { .node_table("Person", "id") .rel_table("KNOWS", "Person", "Person", "src_id", "dst_id") .build()?; - // Each dataset is pinned to the version opened here until `engine.refresh()`. + // Each dataset is pinned to the version opened here until `engine.checkout_latest()`. let sources = HashMap::from([ ( "Person".to_string(), @@ -108,10 +108,10 @@ async fn main() -> lance_csr::Result<()> { // +------------------+ // --8<-- [end:query] - // --8<-- [start:refresh] - // Bob now knows Alice too. The engine doesn't see the write until refresh(). + // --8<-- [start:checkout_latest] + // Bob now knows Alice too. The engine doesn't see the write until checkout_latest(). write_lance(knows(vec![2], vec![1])?, &knows_uri, WriteMode::Append).await?; - engine.refresh().await?; + engine.checkout_latest().await?; let result = engine.execute(query, params()).await?; print_batches(&[result])?; // Alice, Carol, David @@ -121,7 +121,7 @@ async fn main() -> lance_csr::Result<()> { stats.edges, stats.update, stats.version ); // 5 edges, update=delta, version=Some(2) - // --8<-- [end:refresh] + // --8<-- [end:checkout_latest] // --8<-- [start:timeout] // One deadline for both: checks inside nested-loop joins, and a timer that stops diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs index 5450bee..de01402 100644 --- a/crates/lance-csr/src/engine.rs +++ b/crates/lance-csr/src/engine.rs @@ -5,7 +5,7 @@ //! //! [`CypherEngine`] holds a snapshot of every input: the DataFusion context, the //! catalog, join-key statistics and, with `use_csr`, the CSR adjacencies and node -//! indexes. Queries share the snapshot read-only; [`CypherEngine::refresh`] builds a +//! indexes. Queries share the snapshot read-only; [`CypherEngine::checkout_latest`] builds a //! complete replacement and swaps it in. //! //! ```no_run @@ -63,8 +63,8 @@ use crate::{GraphSourceCatalog, InMemoryCatalog}; pub enum EngineSource { /// On-disk Lance dataset, read lazily through `LanceTableProvider`. /// - /// `dataset` is pinned to one version; only `refresh()` replaces it. `uri` and - /// `storage_options` are kept so refresh can reopen the same location with the + /// `dataset` is pinned to one version; only `checkout_latest()` replaces it. `uri` and + /// `storage_options` are kept so `checkout_latest()` can reopen the same location with the /// same credentials. Lance { uri: String, @@ -79,7 +79,7 @@ pub enum EngineSource { impl EngineSource { /// Open a Lance dataset at `uri`, pinned to `version` (`None` means the latest). /// - /// `storage_options` are kept so [`CypherEngine::refresh`] can reopen the same + /// `storage_options` are kept so [`CypherEngine::checkout_latest`] can reopen the same /// location with the same credentials. pub async fn open_lance( uri: impl Into, @@ -172,7 +172,7 @@ pub struct CsrStats { /// Live edges outside the base CSR: appended since its last full build and not /// since removed. Lance's `num_unindexed_rows`, counted in edges. pub unindexed_edges: u64, - /// How the last refresh or `optimize_csr()` produced it: `build` (first), `rebuild` + /// How the last build, `checkout_latest()` or `optimize_csr()` produced it: `build` (first), `rebuild` /// (full), `loaded` (from a stored file, with any changes since applied as a /// delta), `delta` (changes applied) or `reused` (no edge change). pub update: &'static str, @@ -197,11 +197,11 @@ pub struct NodeIndexStats { /// Everything a query needs, built once and shared read-only by concurrent queries. /// -/// `refresh()` builds a complete replacement and swaps it in; queries already running +/// `checkout_latest()` builds a complete replacement and swaps it in; queries already running /// keep the `Arc` they started with, so they finish on their original snapshot. struct EngineState { sources: HashMap, - /// Unoptimized plan templates for this snapshot; a refresh starts an empty cache. + /// Unoptimized plan templates for this snapshot; `checkout_latest()` starts an empty cache. plans: Mutex>, catalog: Arc, context: SessionContext, @@ -674,7 +674,7 @@ fn memtable_partitions(batch: &RecordBatch) -> Vec> { /// Node ids count as unique (the row count). Edge endpoints use the CSR's distinct /// source and target counts when the CSR is built, and otherwise one scan of the two /// endpoint columns; only integer endpoints are counted. Reused while a dataset is -/// the same snapshot, so an unchanged `refresh()` costs nothing. +/// the same snapshot, so an unchanged `checkout_latest()` costs nothing. // shortcut: the fallback scan reads every edge endpoint at construction and on each changed refresh (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time. async fn build_key_statistics( config: &GraphConfig, @@ -900,12 +900,12 @@ async fn build_csr_entries( /// /// Lance datasets are read lazily from storage, pinned to the version they were /// opened at: writes made later (by this or another process) are not visible until -/// [`refresh`](Self::refresh). Each dataset is pinned separately; versions across +/// [`checkout_latest`](Self::checkout_latest). Each dataset is pinned separately; versions across /// tables are not a transactional multi-table snapshot. Arrow batches are held in /// memory and never change; build a new engine to replace them. /// /// Building the CSR (`use_csr`) is CPU-bound work inside an async function: in an -/// application with its own async runtime, construct and refresh the engine on a +/// application with its own async runtime, construct the engine and call `checkout_latest()` on a /// blocking thread (for example `tokio::task::spawn_blocking` plus `block_on`). /// Options for [`CypherEngine::with_options`]. #[derive(Debug, Clone, Default, PartialEq)] @@ -917,7 +917,7 @@ pub struct EngineOptions { /// Store each fully built CSR inside its Lance relationship dataset, at /// `_lance_csr/.arrow`, and load it instead of building when an engine /// starts. Changes since the stored version apply as a delta, under - /// the same 25% threshold as `refresh()`. Ignored without `use_csr` and for Arrow + /// the same 25% threshold as `checkout_latest()`. Ignored without `use_csr` and for Arrow /// input. A failed write is logged and doesn't fail the engine. pub persist_csr: bool, } @@ -926,8 +926,9 @@ pub struct CypherEngine { config: GraphConfig, state: RwLock>, options: EngineOptions, - /// Held for a whole refresh, so a slower refresh can't swap in older data over a newer one. - refresh_lock: futures::lock::Mutex<()>, + /// Held for a whole update of the snapshot, so a slower `checkout_latest()` can't swap in older + /// data over a newer one. + update_lock: futures::lock::Mutex<()>, } impl CypherEngine { @@ -953,7 +954,7 @@ impl CypherEngine { /// Like [`new`](Self::new), with a target for DataFusion query partitioning. /// /// `Some(n)` sets DataFusion's `target_partitions` for every query, kept across - /// [`refresh`](Self::refresh); `None` keeps DataFusion's default (the number of + /// [`checkout_latest`](Self::checkout_latest); `None` keeps DataFusion's default (the number of /// cores). It is not a thread or memory limit. In-memory Arrow inputs are still /// sliced into one partition per core. /// @@ -998,11 +999,11 @@ impl CypherEngine { config, state: RwLock::new(Arc::new(state)), options, - refresh_lock: Default::default(), + update_lock: Default::default(), }) } - /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent refresh. + /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent `checkout_latest()`. fn snapshot(&self) -> Arc { self.state .read() @@ -1087,7 +1088,7 @@ impl CypherEngine { /// /// Batches are computed as the stream is polled. Scans, filters and projections /// stream with bounded memory; joins, sorts and aggregations may still buffer. - /// The stream keeps what it reads alive, so a concurrent `refresh()` does not + /// The stream keeps what it reads alive, so a concurrent `checkout_latest()` does not /// affect it. Planning errors return here; execution errors arrive in the stream. /// Dropping the stream cancels the remaining work. pub async fn execute_stream( @@ -1123,15 +1124,16 @@ impl CypherEngine { vector_search.search(&candidates).await } - /// Move every Lance dataset to its latest version. + /// Move every Lance dataset to its latest version: the engine's counterpart of + /// Lance's `Dataset::checkout_latest()`. /// /// Datasets whose version and commit time are unchanged keep their open handle. /// The replacement is validated in full before it is swapped in; on error nothing /// changes and the previous snapshot stays usable. Queries already running finish /// on the snapshot they started with. Arrow inputs are unchanged. Concurrent calls /// run one at a time. - pub async fn refresh(&self) -> Result<()> { - let _refreshing = self.refresh_lock.lock().await; + pub async fn checkout_latest(&self) -> Result<()> { + let _updating = self.update_lock.lock().await; let current = self.snapshot(); let mut sources = current.sources.clone(); for source in sources.values_mut() { @@ -1153,16 +1155,22 @@ impl CypherEngine { Ok(()) } + /// Former name of [`checkout_latest`](Self::checkout_latest). + #[deprecated(note = "renamed to `checkout_latest`, after Lance's `Dataset::checkout_latest`")] + pub async fn refresh(&self) -> Result<()> { + self.checkout_latest().await + } + /// Rebuild each CSR that carries a delta, at the snapshot the engine is pinned to: the /// CSR's counterpart of Lance's `optimize_indices()`. /// - /// `refresh()` applies changes as a delta over the base CSR and rebuilds on its own + /// `checkout_latest()` applies changes as a delta over the base CSR and rebuilds on its own /// only once they pass 25% of the base. This rebuilds every CSR that carries a delta /// now, so lookups stop carrying it; one without a delta, and everything else, is /// reused. Results never change. The replacement is validated in full before it is - /// swapped in, as for `refresh()`, and the two run one at a time. + /// swapped in, as for `checkout_latest()`, and the two run one at a time. pub async fn optimize_csr(&self) -> Result<()> { - let _refreshing = self.refresh_lock.lock().await; + let _updating = self.update_lock.lock().await; let current = self.snapshot(); let state = build_engine_state( &self.config, @@ -1181,10 +1189,10 @@ impl CypherEngine { /// /// The engine keeps the CSR it has in memory, so queries are unaffected. An /// engine with `persist_csr` stores a new file at its next full build; create - /// engines without it to stay in memory only. Runs one at a time with `refresh()` + /// engines without it to stay in memory only. Runs one at a time with `checkout_latest()` /// and `optimize_csr()`. pub async fn delete_stored_csr(&self) -> Result { - let _refreshing = self.refresh_lock.lock().await; + let _updating = self.update_lock.lock().await; let current = self.snapshot(); let mut deleted = 0; for (_, mapping) in self.config.iter_relationship_mappings() { @@ -1201,7 +1209,7 @@ impl CypherEngine { /// CSR adjacency statistics per relationship type (experimental, `use_csr`). /// - /// An identical map after `refresh()` means the snapshot itself was unchanged. + /// An identical map after `checkout_latest()` means the snapshot itself was unchanged. pub fn csr_stats(&self) -> HashMap { self.snapshot() .csr @@ -1280,7 +1288,7 @@ mod tests { } #[tokio::test] - async fn target_partitions_survive_refresh_without_changing_other_defaults() { + async fn target_partitions_survive_checkout_latest_without_changing_other_defaults() { let defaults = SessionConfig::new(); for target_partitions in [None, Some(1), Some(2)] { let engine = CypherEngine::with_target_partitions( @@ -1292,7 +1300,7 @@ mod tests { .await .unwrap(); let before = engine.snapshot(); - engine.refresh().await.unwrap(); + engine.checkout_latest().await.unwrap(); for snapshot in [before, engine.snapshot()] { let state = snapshot.context.state(); assert_eq!( @@ -1318,7 +1326,7 @@ mod tests { } #[tokio::test] - async fn plan_cache_reuses_bounds_and_resets_on_refresh() { + async fn plan_cache_reuses_bounds_and_resets_on_checkout_latest() { let engine = CypherEngine::new(person_config(), people_engine_sources(), false) .await .unwrap(); @@ -1385,7 +1393,7 @@ mod tests { assert_eq!(first.num_rows(), 1); // A refresh publishes a new snapshot with an empty cache; the old one is untouched. - engine.refresh().await.unwrap(); + engine.checkout_latest().await.unwrap(); assert!(engine.snapshot().plans.lock().unwrap().is_empty()); assert_eq!(cached(), PLAN_CACHE_ENTRIES); } @@ -1434,10 +1442,10 @@ mod tests { .unwrap(); } - /// A refresh that starts while another is in flight waits for it, then publishes + /// A `checkout_latest()` that starts while another is in flight waits for it, then publishes /// the newest version: an older snapshot can never land after a newer one. #[tokio::test] - async fn refresh_waits_for_an_in_flight_refresh() { + async fn checkout_latest_waits_for_an_in_flight_checkout() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().join("person.lance").display().to_string(); write_people(&uri, vec![1], WriteMode::Create).await; @@ -1452,13 +1460,13 @@ mod tests { let engine = Arc::new(CypherEngine::new(config, sources, true).await.unwrap()); // Stand in for refresh A, still building its replacement. - let in_flight = engine.refresh_lock.lock().await; + let in_flight = engine.update_lock.lock().await; let b = tokio::spawn({ let engine = engine.clone(); - async move { engine.refresh().await } + async move { engine.checkout_latest().await } }); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert!(!b.is_finished(), "refresh B ran while A was in flight"); + assert!(!b.is_finished(), "checkout B ran while A was in flight"); // A newer version lands before A finishes; B must publish it, not anything older. write_people(&uri, vec![2], WriteMode::Append).await; diff --git a/crates/lance-csr/tests/test_engine.rs b/crates/lance-csr/tests/test_engine.rs index fe1b18d..5e5adcc 100644 --- a/crates/lance-csr/tests/test_engine.rs +++ b/crates/lance-csr/tests/test_engine.rs @@ -116,7 +116,7 @@ async fn parameters_and_streaming() { } #[tokio::test] -async fn lance_input_is_pinned_until_refresh() { +async fn lance_input_is_pinned_until_checkout_latest() { let dir = tempfile::tempdir().unwrap(); let person_uri = dir.path().join("person.lance"); let knows_uri = dir.path().join("knows.lance"); @@ -144,14 +144,14 @@ async fn lance_input_is_pinned_until_refresh() { ["Carol"] ); - // A write after construction stays invisible until refresh(). + // A write after construction stays invisible until checkout_latest(). write(knows(&[(2, 4)]), knows_uri, WriteMode::Append).await; assert_eq!( names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()), ["Carol"] ); - engine.refresh().await.unwrap(); + engine.checkout_latest().await.unwrap(); assert_eq!( names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()), ["Carol", "David"] @@ -179,7 +179,7 @@ async fn optimize_csr_folds_the_delta_into_the_base() { let engine = CypherEngine::new(config(), sources, true).await.unwrap(); write(knows(&[(2, 4)]), knows_uri, WriteMode::Append).await; - engine.refresh().await.unwrap(); + engine.checkout_latest().await.unwrap(); let csr = engine.csr_stats()["KNOWS"].clone(); assert_eq!( (csr.update, csr.delta_edges, csr.unindexed_edges), diff --git a/docs/AGENTS.md b/docs/AGENTS.md index b295d0e..cd1e3df 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -83,7 +83,7 @@ them. caveats and known issues; `???` makes a collapsible one for long detail; - content tabs (`=== "Tab"`) for alternatives, such as install methods or input types; - code annotations (`# (1)!` plus a numbered list after the block) to explain specific lines; - - Mermaid diagrams (```` ```mermaid ````) for flows: query pipelines, refresh decisions, snapshots; + - Mermaid diagrams (```` ```mermaid ````) for flows: query pipelines, `checkout_latest()` decisions, snapshots; - tables for options and comparisons, definition lists for method references; - grid cards on the home page only. - Plain, direct sentences. Use "you". Avoid filler and marketing adjectives. Spell out a term the first time diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 6b3a416..ebc825d 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -163,10 +163,10 @@ uv run --no-sync python benchmarks/csr_expand.py --data /tmp/csr-bench --edges 1 | Script | Measures | | --- | --- | -| `csr_expand.py` | CSR build cost, single-hop latency against the join path, refresh cost and peak RSS, per edge count | -| `csr_delta.py` | `refresh()` with a delta against a full rebuild, after one append or delete commit (uses `csr_expand.py`'s data) | +| `csr_expand.py` | CSR build cost, single-hop latency against the join path, `checkout_latest()` cost and peak RSS, per edge count | +| `csr_delta.py` | `checkout_latest()` with a delta against a full rebuild, after one append or delete commit (uses `csr_expand.py`'s data) | | `csr_delta_accumulated.py` | Refresh cost as deltas accumulate over many small commits (the basis of the 25% rebuild threshold) | -| `engine_cache.py` | Engine construction, first-query, warm-query and refresh timings; can compare against upstream `lance_graph` | +| `engine_cache.py` | Engine construction, first-query, warm-query and `checkout_latest()` timings; can compare against upstream `lance_graph` | | `bounded_memory.py` | Peak memory of streamed scans over Lance data larger than a memory cap (`generate`, then `run`) | Each script's docstring lists its options. When you change performance-sensitive code, include before and diff --git a/docs/csr.md b/docs/csr.md index 74aa3dc..d6e0c80 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -99,7 +99,7 @@ Node index During query execution, each CSR expansion partition keeps one input batch and collects at most one output chunk of neighbour positions before fetching properties. It yields that chunk before visiting the remaining neighbours, even for a single high-degree node. Deleted edge positions are read from the - shared snapshot rather than copied for each query; refresh resolves those positions by scanning the + shared snapshot rather than copied for each query; `checkout_latest()` resolves those positions by scanning the affected sources' neighbour lists. Cancellation checkpoints count source rows and frontiers with no surviving edges, @@ -273,7 +273,7 @@ print(engine.csr_stats()["KNOWS"]) | `delta_edges` | Appended edges held outside the base CSR since its last full build. | | `tombstones` | Removed edges masked out since the last full build. | | `unindexed_edges` | Live edges outside the base CSR: appended since its last full build and not since removed. Lance index statistics call this `num_unindexed_rows`. | -| `update` | How the last build, `refresh()` or [`optimize_csr()`](#rebuilding-the-csr) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). | +| `update` | How the last build, `checkout_latest()` or [`optimize_csr()`](#rebuilding-the-csr) produced it: `build` (first), `rebuild` (full), `loaded` (from a [stored file](#storing-the-csr), with any changes since applied as a delta), `delta` (changes applied) or `reused` (no edge change). | | `version` | The relationship dataset's Lance version, or `None` for PyArrow input. | | `base_version` | The version of the last full build, or `None` for PyArrow input. It trails `version` while deltas apply. | | `build_ms` | Wall time of that build or delta, both directions. | @@ -298,30 +298,37 @@ each other, but it also means the CSR falls behind as soon as anyone writes to t Two calls bring it back up to date, and they do different jobs: -1. **[`refresh()`](#refresh)** moves the engine to the newest version of each Lance dataset. Instead of rebuilding +1. **[`checkout_latest()`](#checkout-latest)** moves the engine to the newest version of each Lance dataset. Instead of rebuilding the CSR, it records the edges added or removed since the last build as a *delta*, a small set of changes kept next to the CSR. Call it whenever your queries should see new writes. 2. **[`optimize_csr()`](#rebuilding-the-csr)** rebuilds a CSR from scratch, so it no longer needs its delta. Call it occasionally, once enough changes have built up that you'd rather lookups stopped working through them. +The two calls aren't a matched pair, though. `checkout_latest()` isn't really a CSR operation: it's named after, and +works like, Lance's own `LanceDataset.checkout_latest()`, and it moves the whole engine to the newest version of every +Lance dataset it reads. Node tables, node indexes and the statistics used to plan joins all move with it, and keeping +the CSR in step is just one part of that, done as cheaply as possible with a delta. `optimize_csr()` is the call that's +about the CSR itself: it rebuilds a CSR in a running engine when you decide, rather than waiting for the delta to get +too large. It plays the part Lance's `optimize_indices()` plays for Lance's own indexes. + If you gave the engine in-memory PyArrow tables instead of Lance datasets, neither call applies: those tables can't change underneath the engine, so their CSR is never out of date. To query new in-memory data, create a new engine. -### `refresh()`: pick up new writes { #refresh } +### `checkout_latest()`: pick up new writes { #checkout-latest } Because each Lance dataset can evolve independently as you write to it, the engine has to be told that a newer -version is available. `refresh()` does that: it checks every Lance dataset for a newer version and moves the engine to +version is available. `checkout_latest()` does that: it checks every Lance dataset for a newer version and moves the engine to it. Call it after the writes your queries should see, whether they came from `pylance`, LanceDB or another process. -If one writer updates several tables, wait until all of them have committed before you refresh, since each Lance +If one writer updates several tables, wait until all of them have committed before you call it, since each Lance dataset is pinned on its own (see [snapshots are per Lance dataset](lance.md#lazy-version-pinned-reads)). When nothing -has changed, a refresh keeps what the engine already has and returns quickly, so calling it on a timer or before +has changed, `checkout_latest()` keeps what the engine already has and returns quickly, so calling it on a timer or before each batch of queries is fine. -Rebuilding the whole CSR on every refresh would mean scanning every edge again, even when only a handful changed. -So `refresh()` reads only the new parts of each relationship dataset and applies the changes on top of the CSR from +Rebuilding the whole CSR on every call would mean scanning every edge again, even when only a handful changed. +So `checkout_latest()` reads only the new parts of each relationship dataset and applies the changes on top of the CSR from the last full build, which the rest of this page calls the *base*. Appended edges go into the delta, and deleted edges are marked so that lookups skip them; `csr_stats()` counts those marks as `tombstones`. Answers are exactly -what a full rebuild would give. To see what a refresh did, check `update` in [`csr_stats()`](#csr_stats): `reused` +what a full rebuild would give. To see what a call did, check `update` in [`csr_stats()`](#csr_stats): `reused` means nothing changed, `delta` means the changes were applied, and `rebuild` means the CSR was built again from scratch, which happens in the cases below. @@ -338,9 +345,9 @@ scratch, which happens in the cases below. use_csr=True, ) - # David now knows Alice. The engine doesn't see the write until refresh(). + # David now knows Alice. The engine doesn't see the write until checkout_latest(). lance.write_dataset(pa.table({"src": [4], "dst": [1]}), "knows.lance", mode="append") - engine.refresh() + engine.checkout_latest() stats = engine.csr_stats()["KNOWS"] print(stats["edges"], stats["delta_edges"], stats["update"]) # 5 1 delta @@ -351,19 +358,19 @@ scratch, which happens in the cases below. This continues the on-disk example from [Using from Rust](rust.md), where Bob gains an edge to Alice. ```rust - --8<-- "crates/lance-csr/examples/embedded_query.rs:refresh" + --8<-- "crates/lance-csr/examples/embedded_query.rs:checkout_latest" ``` -??? info "How `refresh()` decides" +??? info "How `checkout_latest()` decides" ```mermaid flowchart LR - R["refresh()"] -- "no edge changes" --> U["reused"] + R["checkout_latest()"] -- "no edge changes" --> U["reused"] R -- "changes under 25% of the base" --> DL["delta"] R -- "25% or more, or new endpoint columns" --> RB["rebuild"] ``` - - A refresh rebuilds a CSR instead of adding to its delta once the changes since the last full build would reach + - `checkout_latest()` rebuilds a CSR instead of adding to its delta once the changes since the last full build would reach 25% of the base, or when the relationship dataset was overwritten or its endpoint columns changed. The threshold is fixed: every lookup reads the delta as well as the base, so a large delta slows lookups down, and a rebuild clears it. `python/benchmarks/csr_delta_accumulated.py` measures that trade-off. @@ -376,23 +383,23 @@ scratch, which happens in the cases below. ### `optimize_csr()`: rebuild to clear the delta { #rebuilding-the-csr } -The delta keeps refreshes cheap, but every lookup has to check it as well as the base. A refresh only rebuilds once +The delta keeps `checkout_latest()` cheap, but every lookup has to check it as well as the base. It only rebuilds once the delta reaches 25% of the base, so a smaller delta can stay around indefinitely and slow lookups a little the whole time. `optimize_csr()` lets you clear it at a moment you choose. It rebuilds every CSR that has a delta, from a scan of all its edges, at the version the engine is already pinned to, and leaves CSRs without a delta alone. That costs about as much as building the CSR the first time, so it's best kept for after a bulk load, before a run of -latency-sensitive queries, or a quiet period, rather than called after every refresh. Query results are the same +latency-sensitive queries, or a quiet period, rather than called after every `checkout_latest()`. Query results are the same before and after. ```python -engine.refresh() # applies the new edges as a delta +engine.checkout_latest() # applies the new edges as a delta engine.optimize_csr() # rebuilds the CSR, now including those edges stats = engine.csr_stats()["KNOWS"] print(stats["update"], stats["unindexed_edges"]) # rebuild 0 ``` Afterwards, `csr_stats()` shows `delta_edges`, `tombstones` and `unindexed_edges` at 0, and `base_version` matches -`version`. `optimize_csr()` doesn't look for newer data, so call `refresh()` first if the rebuild should include the +`version`. `optimize_csr()` doesn't look for newer data, so call `checkout_latest()` first if the rebuild should include the latest writes. ??? info "How this maps onto Lance's indexes" @@ -413,18 +420,18 @@ latest writes. ### Which one to call -!!! tip "`refresh()` or `optimize_csr()`?" +!!! tip "`checkout_latest()` or `optimize_csr()`?" - In short, `refresh()` keeps the engine up to date at little cost, and `optimize_csr()` now and then clears the - delta that refreshes leave behind. + In short, `checkout_latest()` keeps the engine up to date at little cost, and `optimize_csr()` now and then clears the + delta that `checkout_latest()` leaves behind. - | | `refresh()` | `optimize_csr()` | + | | `checkout_latest()` | `optimize_csr()` | | --- | --- | --- | | Data | Moves to the latest version of each Lance dataset | Stays at the version the engine is pinned to | | Usual effect on a CSR | Applies the new writes as a delta, reading only new fragments | Rebuilds it from a scan of all its edges | | Rebuilds a CSR when | Its delta would pass 25% of the base | It carries any delta at all | | Cost | In proportion to the new writes | Like the first build | - | When to call it | Whenever queries should see new writes | Now and then, after `refresh()`, once a delta has built up | + | When to call it | Whenever queries should see new writes | Now and then, after `checkout_latest()`, once a delta has built up | Whenever either call rebuilds a CSR, it runs the same full build and ends up with the same CSR. Both build the replacement completely before the engine switches to it, and they never run at the same time. @@ -458,10 +465,10 @@ The file belongs to the table: dropping or copying the dataset drops or copies i files it recognises. - **Changes since the stored version** apply as a delta, read from the new fragments only, exactly as - [`refresh()`](#keeping-the-csr-current) applies them. `update` says `loaded` and `base_version` names the + [`checkout_latest()`](#keeping-the-csr-current) applies them. `update` says `loaded` and `base_version` names the stored version. That holds after compaction and `cleanup_old_versions()` too. - **Past the 25% threshold**, the engine rebuilds the CSR and stores the new version. Every full build is stored, - including those from `refresh()` and [`optimize_csr()`](#rebuilding-the-csr), and older files are deleted + including those from `checkout_latest()` and [`optimize_csr()`](#rebuilding-the-csr), and older files are deleted once a newer one is written. - **A file that can't be used** (corrupt, written for other endpoint columns or by another format) is ignored and rebuilt, so a bad file costs a build, never a wrong answer. diff --git a/docs/cypher.md b/docs/cypher.md index d47a1c5..f58bfc9 100644 --- a/docs/cypher.md +++ b/docs/cypher.md @@ -6,7 +6,7 @@ There are two ways to run Cypher, and most code wants the first: | --- | --- | --- | | Use it for | Many queries over the same data | One query, once | | Setup cost | Paid once, in the constructor | Paid on every `execute()` | -| Lance datasets | Read lazily, pinned to a version, `refresh()` | Read into memory | +| Lance datasets | Read lazily, pinned to a version, `checkout_latest()` | Read into memory | | CSR index | `use_csr=True` | Not available | | Streaming results | `execute_batches()` | No | @@ -126,7 +126,7 @@ print(rows) # 4 - CSR expansion visits neighbours and fetches their properties a chunk at a time, including when one node has many neighbours. See [CSR memory cost](csr.md#turning-it-on) for what it keeps in memory. - Planning errors (a syntax error, an unknown label) raise immediately. Execution errors raise while reading. -- The reader keeps its snapshot alive, so a concurrent [`refresh()`](lance.md#refresh) doesn't affect it. +- The reader keeps its snapshot alive, so a concurrent [`checkout_latest()`](lance.md#checkout-latest) doesn't affect it. - Dropping the reader cancels the remaining work. ### Query deadlines @@ -163,7 +163,7 @@ print("CsrExpandExec" in plan) # True | Method | Returns | | --- | --- | | `config()` | The engine's `GraphConfig`. | -| `refresh()` | `None`. Moves every Lance dataset to its latest version. See [Lance datasets](lance.md#refresh). | +| `checkout_latest()` | `None`. Moves every Lance dataset to its latest version. See [Lance datasets](lance.md#checkout-latest). | | `dataset_versions()` | `dict[str, int]`: the pinned version of each Lance dataset. | | `csr_stats()`, `node_index_stats()` | CSR statistics. See [The CSR index](csr.md#inspecting-the-csr). | | `execute_with_vector_rerank(query, vector_search)` | A table reranked by vector similarity. See [Vector search](vector-search.md). | @@ -372,7 +372,7 @@ in `__lance_csr_identity`. Relationship records also carry the node records `__l `__lance_csr_end`. These metadata fields are distinct from user properties, even if a property has the same name. `keys(entity)` lists only non-null user properties; `keys(map)` includes null-valued keys. Prefer the Cypher property and endpoint functions when writing queries. Relationship identities identify actual occurrences within this query's inventory, not persistent -IDs that can be compared across queries or refreshes. Rich paths support the same signed integer and +IDs that can be compared across queries or snapshots. Rich paths support the same signed integer and `uint8`/`uint16`/`uint32` keys as the CSR. Mixed labels may share compatible properties; conflicting Arrow property types fail explicitly. Non-null node IDs must be unique within each participating label so relationship endpoints identify one node; duplicate IDs raise an error. Expanding a graph pattern from @@ -382,7 +382,7 @@ A path value needs a query-local inventory of the participating node and relatio properties. It therefore materializes those tables, even with `use_csr=True`. Traversal output is emitted in batches and named ordinary paths keep a cursor per hop, but inventory memory grows with the graph. Rich path queries bypass the query-template cache: each execution binds its current parameters and -creates an inventory from its own snapshot. A reader opened before `refresh()` retains the old snapshot. +creates an inventory from its own snapshot. A reader opened before `checkout_latest()` retains the old snapshot. Queries asking only for shortest-path length, node keys or path null checks, including homogeneous node-property comprehensions and paths carried through `WITH` aliases, retain the usual CSR execution route when the relationship joins the same node label. Cross-label shortest paths use typed identities even for diff --git a/docs/execution.md b/docs/execution.md index c4df547..4f8ca27 100644 --- a/docs/execution.md +++ b/docs/execution.md @@ -71,7 +71,7 @@ make it build a multi-million-row intermediate as one single-threaded hash table lance-csr computes distinct counts for every node key and edge endpoint column (from the CSR when there is one, otherwise with one scan of the endpoint columns) and reports them to DataFusion's join planner. The -counts are cached per dataset version, so an unchanged `refresh()` costs nothing. +counts are cached per dataset version, so an unchanged `checkout_latest()` costs nothing. ## Parallelism @@ -93,14 +93,14 @@ sequenceDiagram participant E as CypherEngine participant Q2 as Query 2 Q1->>E: execute() takes snapshot v1 - Note over E: refresh() builds snapshot v2
(reusing unchanged parts), then swaps it in + Note over E: checkout_latest() builds snapshot v2
(reusing unchanged parts), then swaps it in Q2->>E: execute() takes snapshot v2 E-->>Q1: finishes on v1 E-->>Q2: finishes on v2 ``` -`refresh()` builds a complete replacement next to the current snapshot, reusing everything whose source -dataset hasn't changed, and swaps it in atomically. A failed refresh leaves the old snapshot in place. +`checkout_latest()` builds a complete replacement next to the current snapshot, reusing everything whose source +dataset hasn't changed, and swaps it in atomically. A failed call leaves the old snapshot in place. ## Single-machine scope diff --git a/docs/getting-started.md b/docs/getting-started.md index d6028ba..3234d79 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -214,7 +214,7 @@ than one query against the same data. - [Graph schema](graph-schema.md): every option of `GraphConfig`. - [Querying with Cypher](cypher.md): parameters, streaming, `explain()` and the supported Cypher. -- [Lance datasets](lance.md): query data on disk without loading it, and pick up new writes with `refresh()`. +- [Lance datasets](lance.md): query data on disk without loading it, and pick up new writes with `checkout_latest()`. - [Using from Rust](rust.md): the same engine as a Rust library. - The [`examples/`](https://github.com/hyperdimensionalcomputingai/lance-csr/tree/main/examples) directory has runnable scripts. diff --git a/docs/index.md b/docs/index.md index a27133b..ae01b96 100644 --- a/docs/index.md +++ b/docs/index.md @@ -78,7 +78,7 @@ each relationship hop from a hash join into an array lookup, which is what makes --- - Lazy, version-pinned reads, `refresh()`, streaming results and LanceDB tables. + Lazy, version-pinned reads, `checkout_latest()`, streaming results and LanceDB tables. [:octicons-arrow-right-24: Lance datasets](lance.md) diff --git a/docs/internals/index.md b/docs/internals/index.md index 2532ed4..abb3059 100644 --- a/docs/internals/index.md +++ b/docs/internals/index.md @@ -53,7 +53,7 @@ matters. - **Bounded, streaming operators.** The extension operators yield bounded output batches and resume where they left off, even inside a single high-degree node's neighbour list, and they check for cancellation as they go. - **Immutable snapshots.** An engine holds a snapshot of pinned dataset versions plus the indexes built for them. - `refresh()` builds a replacement next to it, reusing whatever didn't change, and swaps it in atomically, so running + `checkout_latest()` builds a replacement next to it, reusing whatever didn't change, and swaps it in atomically, so running queries never see a half-updated index. ## Where the extensions plug in diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md index 6d23cc8..43aa2d9 100644 --- a/docs/internals/indexes.md +++ b/docs/internals/indexes.md @@ -2,7 +2,7 @@ The [custom operators](operators.md) read two in-memory indexes. Both are built from the same underlying Lance dataset and point back into it, so neither holds a copy of node or relationship properties. [The CSR index](../csr.md) covers what users see: -how to turn it on, `csr_stats()`, and when to call `refresh()` and `optimize_csr()`. This page covers how the indexes +how to turn it on, `csr_stats()`, and when to call `checkout_latest()` and `optimize_csr()`. This page covers how the indexes are laid out and maintained. The compact store of offsets and neighbours came from [lance-graph](https://github.com/lance-format/lance-graph), @@ -67,7 +67,7 @@ table, so edge commits update only the adjacency and node commits rebuild only t ## How a refresh diffs Lance versions -`refresh()` compares the pinned and latest versions of each relationship dataset fragment by fragment, from their +`checkout_latest()` compares the pinned and latest versions of each relationship dataset fragment by fragment, from their manifests: | Fragment | Result | @@ -133,7 +133,7 @@ names, their Lance field ids and types, and the fragments the base was built fro When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as a refresh diffs two versions (see [How a refresh diffs Lance versions](#how-a-refresh-diffs-lance-versions)), so the file's own version doesn't need to exist any more. An empty diff uses the file as it is; otherwise the changes become the delta, under the same 25% -threshold. Rebuilds after start-up (from `refresh()` or `optimize_csr()`) don't look for a file, since the stored base +threshold. Rebuilds after start-up (from `checkout_latest()` or `optimize_csr()`) don't look for a file, since the stored base is the one the engine already holds, but they do store the new one. Every batch is compressed with zstd, Arrow IPC's own buffer compression, on top of the packing. A load decompresses diff --git a/docs/internals/optimizer.md b/docs/internals/optimizer.md index 1bec696..5fe00af 100644 --- a/docs/internals/optimizer.md +++ b/docs/internals/optimizer.md @@ -114,4 +114,4 @@ graph join chain, that turns a multi-million-row intermediate into an estimate o `KeyStatisticsProvider` wraps each table and reports distinct counts for its node-id and edge-endpoint columns, taken from the CSR when there is one and otherwise from one scan of the endpoint columns. Every other statistic comes from -the inner table. The counts are cached per dataset version, so a `refresh()` with no changes costs nothing. +the inner table. The counts are cached per dataset version, so a `checkout_latest()` with no changes costs nothing. diff --git a/docs/lance.md b/docs/lance.md index e24527f..9c1c698 100644 --- a/docs/lance.md +++ b/docs/lance.md @@ -36,21 +36,25 @@ print(engine.dataset_versions()) # {'Person': 1, 'KNOWS': 1} ``` Each dataset is **pinned to the version you passed in**. Writes made later, by your process or another one, -stay invisible until you call `refresh()`. That gives every query a stable snapshot. +stay invisible until you call `checkout_latest()`. That gives every query a stable snapshot. !!! note "Snapshots are per dataset" Each dataset is pinned separately. The versions across tables are not a transactional multi-table - snapshot; if a writer updates `Person` and `KNOWS` one after the other, refresh after both commits. + snapshot; if a writer updates `Person` and `KNOWS` one after the other, call `checkout_latest()` after both commits. !!! warning "Use pylance 13.x to write" The engine is built against Lance 13. Datasets written by an older or newer major version of `pylance` may use a file format it can't read. Installing `lance-csr` pins a compatible `pylance` for you. -## `refresh()` { #refresh } +## `checkout_latest()` { #checkout-latest } -`refresh()` moves every Lance dataset to its latest version: +`checkout_latest()` moves every Lance dataset to its latest version. It's named after Lance's own +`LanceDataset.checkout_latest()`, which does the same for a single dataset, and it updates everything the engine +built from that data along with it: node indexes, join statistics and, with `use_csr=True`, the CSR. It isn't a +CSR-specific call; to rebuild a CSR, use [`optimize_csr()`](csr.md#rebuilding-the-csr) (see +[Keeping the CSR current](csr.md#keeping-the-csr-current)): ```python lance.write_dataset( @@ -59,27 +63,32 @@ lance.write_dataset( lance.write_dataset(pa.table({"src": [3], "dst": [4]}), "knows.lance", mode="append") print(engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 3}] -engine.refresh() +engine.checkout_latest() print(engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 4}] print(engine.dataset_versions()) # {'Person': 2, 'KNOWS': 2} ``` -What `refresh()` guarantees: +!!! note "Formerly `refresh()`" + + This method was called `refresh()` before. `refresh()` still works, and does exactly the same, but it + warns with `DeprecationWarning` and will be removed in a later release. + +What `checkout_latest()` guarantees: - **Unchanged datasets keep their handle.** A dataset whose version and commit time are unchanged is reused, - along with its CSR, node index and statistics, so a no-op refresh is cheap. -- **Schema changes are picked up.** New columns are queryable after the refresh. + along with its CSR, node index and statistics, so calling it when nothing changed is cheap. +- **Schema changes are picked up.** New columns are queryable afterwards. - **All or nothing.** The replacement is validated in full before it's swapped in. On error, nothing changes and the previous snapshot keeps working. -- **Concurrent refreshes cannot overwrite a newer published snapshot with an older candidate.** If another - refresh publishes first, the overlapping refresh retries from that snapshot and reopens the latest datasets. - The retry can add work when refreshes overlap; it does not block queries while the replacement is built. -- **Running queries are unaffected.** Queries and `execute_batches()` readers that started before the refresh +- **Concurrent calls cannot overwrite a newer published snapshot with an older candidate.** If another call + publishes first, the overlapping one retries from that snapshot and reopens the latest datasets. The retry + can add work when calls overlap; it does not block queries while the replacement is built. +- **Running queries are unaffected.** Queries and `execute_batches()` readers that started before the call finish on their original snapshot. - **The CSR updates incrementally**, as described in [Keeping the CSR current](csr.md#keeping-the-csr-current). - **PyArrow inputs don't change.** To replace in-memory data, build a new engine. -A refresh reopens each dataset by its URI with the same `storage_options` it was opened with, so object-store +`checkout_latest()` reopens each dataset by its URI with the same `storage_options` it was opened with, so object-store credentials carry over. ## Repeated queries @@ -91,8 +100,8 @@ rows and results are never cached. Query text and parameter names/types identify separate from non-null values. Least recently used templates are evicted at the limit. Lists, maps and parameterized `SKIP`/`LIMIT` counts use ordinary planning because their shape or value can -change the plan. Queries whose operators cannot produce a reusable template fall back to ordinary planning. `explain()` plans the current values directly. A successful `refresh()` starts an empty cache for the -new snapshot, including a refresh whose datasets did not change; active readers retain their existing plans. +change the plan. Queries whose operators cannot produce a reusable template fall back to ordinary planning. `explain()` plans the current values directly. A successful `checkout_latest()` starts an empty cache for the +new snapshot, even when no dataset changed; active readers retain their existing plans. These rules apply with and without CSR. They do not promise a speedup for a particular workload. ## Query parallelism @@ -108,13 +117,13 @@ serial_engine = CypherEngine( target_partitions=1, ) print(serial_engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{'n': 4}] -serial_engine.refresh() # retains target_partitions=1 +serial_engine.checkout_latest() # retains target_partitions=1 ``` The target is a planning preference, not a thread or memory limit; operators may use a different number of partitions. A lower target can help short queries while making larger joins and scans slower. Compare your full workload in the storage mode you use before choosing a value. The default remains unchanged, -and `refresh()` preserves your choice. Booleans and non-integers other than `None` raise `TypeError`; +and `checkout_latest()` preserves your choice. Booleans and non-integers other than `None` raise `TypeError`; zero, negative and out-of-range integers raise `ValueError`. ## Streaming large results @@ -142,7 +151,7 @@ memory. Joins, sorts and aggregations may still buffer their inputs. | Join-key statistics | In memory, a few numbers per key column | | Query results | In memory with `execute()`; streamed with `execute_batches()` | -The CSR and node index are built from the Lance data once and updated on `refresh()`; queries don't rescan +The CSR and node index are built from the Lance data once and updated on `checkout_latest()`; queries don't rescan the edge table to traverse it. ## Tables created with LanceDB @@ -166,12 +175,12 @@ config = ( engine = CypherEngine(config, {"Person": people.to_lance(), "KNOWS": knows.to_lance()}) knows.add(new_edges) # keep writing with the LanceDB API -engine.refresh() # the engine sees the new version +engine.checkout_latest() # the engine sees the new version ``` - Reads are lazy and version-pinned exactly as with `lance.dataset(...)`. - Passing the LanceDB `Table` object itself raises a `TypeError`. `table.to_arrow()` works, but loads the - whole table into memory and never refreshes. + whole table into memory and never sees later versions. - Tested with LanceDB 0.40.0 on local tables (Lance file format 2.2). A LanceDB release bundling a newer Lance major may write files this engine can't read. @@ -205,4 +214,4 @@ print(namespace.base_uri) # graph ``` `CypherQuery` (with a namespace or a dict) reads the data for each `execute()`; use `CypherEngine` for lazy -reads, refresh and the CSR. +reads, `checkout_latest()` and the CSR. diff --git a/docs/rust.md b/docs/rust.md index 32bcd14..a39aa19 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -58,7 +58,7 @@ Each input is an `EngineSource`: `EngineSource::open_lance(uri, version, storage_options)` : Opens a Lance dataset, pinned to `version` (`None` means the latest). The dataset is read lazily as - queries need it, and `storage_options` are kept so `refresh()` can reopen it with the same credentials. + queries need it, and `storage_options` are kept so `checkout_latest()` can reopen it with the same credentials. See [Lance datasets](lance.md). `EngineSource::Arrow(batch)` @@ -92,11 +92,11 @@ Each query runs on the snapshot that was current when it started. ## Pick up new writes -`refresh()` moves every Lance dataset to its latest version and updates the CSR, applying the new edges as a +`checkout_latest()` moves every Lance dataset to its latest version and updates the CSR, applying the new edges as a delta when it can (see [Keeping the CSR current](csr.md#keeping-the-csr-current)): ```rust ---8<-- "crates/lance-csr/examples/embedded_query.rs:refresh" +--8<-- "crates/lance-csr/examples/embedded_query.rs:checkout_latest" ``` `csr_stats()` and `node_index_stats()` return the same fields as in Python, as `CsrStats` and `NodeIndexStats` @@ -118,7 +118,7 @@ different exception depending on the variant: ## Async runtimes and cancellation -- Building the CSR in `new()` and `refresh()` is CPU-heavy work inside an async function, so it holds a +- Building the CSR in `new()` and `checkout_latest()` is CPU-heavy work inside an async function, so it holds a runtime worker thread until it finishes. In an application with its own runtime (a desktop or web backend), run those two calls with `tokio::task::spawn_blocking` so other tasks keep running. - Dropping a query's future, or the stream from `execute_stream`, cancels the rest of that query. @@ -147,7 +147,7 @@ The deadline applies only to work inside the call; it isn't stored in the engine | a PyArrow table in `datasets` | `EngineSource::Arrow(batch)` | | `engine.execute(query, params)` → `pyarrow.Table` | `engine.execute(query, params).await` → `RecordBatch` | | `engine.execute_batches(...)` → `RecordBatchReader` | `engine.execute_stream(...).await` → stream | -| `engine.refresh()` | `engine.refresh().await` | +| `engine.checkout_latest()` | `engine.checkout_latest().await` | | `engine.optimize_csr()` | `engine.optimize_csr().await` | | `engine.execute(query, timeout=2.0)` | `timeout_at(deadline, with_execution_deadline(Some(deadline), engine.execute(...)))` | | `CypherEngine(..., target_partitions=4)` | `CypherEngine::with_target_partitions(config, sources, use_csr, Some(4)).await` | diff --git a/docs/sql.md b/docs/sql.md index fd5bf8d..483453c 100644 --- a/docs/sql.md +++ b/docs/sql.md @@ -55,7 +55,7 @@ print(friends.to_pylist()) The SQL dialect is DataFusion's; see the [DataFusion SQL reference](https://datafusion.apache.org/user-guide/sql/index.html) for what it supports. -`SqlEngine` has no `refresh()` or CSR: those are `CypherEngine` features. +`SqlEngine` has no `checkout_latest()` or CSR: those are `CypherEngine` features. ## From Cypher to SQL diff --git a/examples/README.md b/examples/README.md index de2e6d9..6d61a34 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,7 +8,7 @@ dependencies): ```bash uv run --with lance-csr python examples/basic_cypher.py uv run --with lance-csr python examples/kg_traversal.py -uv run --with lance-csr python examples/lance_snapshot_refresh.py +uv run --with lance-csr python examples/lance_snapshots.py ``` With a local build, so the examples use your changes, set up `python/.venv` as in @@ -35,8 +35,8 @@ uv run --with lance-csr python examples/basic_cypher.py uv run --with lance-csr python examples/kg_traversal.py ``` -- `lance_snapshot_refresh.py`: lazy Lance datasets, snapshots and `refresh()`. +- `lance_snapshots.py`: lazy Lance datasets, snapshots and `checkout_latest()`. ```bash -uv run --with lance-csr python examples/lance_snapshot_refresh.py +uv run --with lance-csr python examples/lance_snapshots.py ``` diff --git a/examples/lance_snapshot_refresh.py b/examples/lance_snapshots.py similarity index 84% rename from examples/lance_snapshot_refresh.py rename to examples/lance_snapshots.py index efae887..333afa0 100644 --- a/examples/lance_snapshot_refresh.py +++ b/examples/lance_snapshots.py @@ -1,11 +1,11 @@ """ -Disk-backed CypherEngine: pinned snapshots, refresh, and streamed results. +Disk-backed CypherEngine: pinned snapshots, checkout_latest(), and streamed results. Requirements: an installed lance-csr wheel (or `maturin develop` in python/), which pulls in a compatible pylance 13.x. Run: - python examples/lance_snapshot_refresh.py + python examples/lance_snapshots.py """ from __future__ import annotations @@ -46,7 +46,7 @@ def main() -> None: "Person": lance.dataset(person_uri), "KNOWS": lance.dataset(root / "knows.lance"), }, - # Optional: choose this target after measuring your own queries; refresh keeps it. + # Optional: choose this target after measuring your own queries; checkout_latest() keeps it. target_partitions=2, ) count = "MATCH (p:Person) RETURN count(*) AS n" @@ -59,13 +59,13 @@ def main() -> None: ).to_pylist(), ) - # Writes are not visible until refresh(): the engine is pinned to version 1. + # Writes are not visible until checkout_latest(): the engine is pinned to version 1. lance.write_dataset(people(ROWS, ROWS + 10), person_uri, mode="append") lance.dataset(person_uri).add_columns({"age": "id % 100"}) # schema change - print("after writes, before refresh:", engine.execute(count).column(0)[0].as_py()) - engine.refresh() + print("after writes, before checkout_latest():", engine.execute(count).column(0)[0].as_py()) + engine.checkout_latest() print( - "after refresh:", + "after checkout_latest():", engine.execute(count).column(0)[0].as_py(), engine.dataset_versions(), ) diff --git a/python/README.md b/python/README.md index 39a051d..471d200 100644 --- a/python/README.md +++ b/python/README.md @@ -110,11 +110,11 @@ lets relationship-type alternation (`[:likes_post|likes_comment]`) pick the righ `with_node_label` and `with_relationship`, the names in lance-csr 0.1, were removed in 0.5.0; use `node_table` and `rel_table` instead. -### 3. Lance datasets: snapshots, refresh and streaming +### 3. Lance datasets: snapshots, checking out new versions and streaming `CypherEngine` reads Lance datasets lazily from storage (projections and filters are pushed into the scan). Each dataset is pinned to the version you pass in; later writes are visible only after -`refresh()`. Use `execute_batches()` to stream large results instead of materializing a table. +`checkout_latest()`. Use `execute_batches()` to stream large results instead of materializing a table. ```python import lance @@ -124,14 +124,14 @@ config = GraphConfig.builder().node_table("Person", "id").build() engine = CypherEngine(config, {"Person": lance.dataset("people.lance")}) engine.execute("MATCH (p:Person) RETURN count(*)") # sees the pinned version -engine.refresh() # move to the latest version +engine.checkout_latest() # move to the latest version for batch in engine.execute_batches("MATCH (p:Person) RETURN p.id"): ... # pyarrow.RecordBatch, one at a time ``` Experimental: `CypherEngine(config, datasets, use_csr=True)` answers single-hop patterns over integer ids with an in-memory CSR adjacency, and fetches target rows by position instead of joining. The CSR is -rebuilt on `refresh()` when the relationship table changes, and the per-label id index when its node +rebuilt on `checkout_latest()` when the relationship table changes, and the per-label id index when its node table changes. Results are identical; peak memory grows by about 95 bytes per edge while the CSR is built (both directions). Write Lance data with `pylance` 13.x: the engine is built against Lance 13 and older or newer @@ -158,11 +158,11 @@ config = ( engine = CypherEngine(config, {"Person": people.to_lance(), "KNOWS": knows.to_lance()}) knows.add(new_edges) # keep writing with the LanceDB API -engine.refresh() # the engine sees the new version +engine.checkout_latest() # the engine sees the new version ``` - Reads are lazy and pinned to the version you pass in, exactly as with `lance.dataset(...)`. -- `table.to_arrow()` also works, but loads the whole table into memory and never refreshes. +- `table.to_arrow()` also works, but loads the whole table into memory and never sees later versions. - Tested with LanceDB 0.40.0 on local tables, which write Lance file format 2.2. A LanceDB release that bundles a newer Lance major may write files this engine cannot read. - Not supported: LanceDB Cloud (remote tables have no `to_lance()`), and tables opened through a diff --git a/python/benchmarks/csr_delta.py b/python/benchmarks/csr_delta.py index e74ccbe..427bab7 100644 --- a/python/benchmarks/csr_delta.py +++ b/python/benchmarks/csr_delta.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""Incremental CSR maintenance: refresh-with-delta cost against a full rebuild. +"""Incremental CSR maintenance: ``checkout_latest()``-with-delta cost against a full rebuild. Uses the graphs written by ``csr_expand.py`` (``--data`` must point at the same directory). For each scenario, on a fresh copy of the edge table (1M-row fragments, plus a ``seq`` row-number column for targeted deletes): - apply one commit (append or delete); -- time ``refresh()`` on a CSR engine built before the commit (delta path); +- time ``checkout_latest()`` on a CSR engine built before the commit (delta path); - time a full CSR build on the new version (what a rebuild costs); - compare warm query latency with the delta in place against the clean build, checking both return identical results. @@ -116,7 +116,7 @@ def main() -> None: engine = CypherEngine(CONFIG, datasets, use_csr=True) change(uri, nodes) - refresh_ms, _ = timed(engine.refresh) + refresh_ms, _ = timed(engine.checkout_latest) stats = engine.csr_stats()["KNOWS"] latest = {"Person": person, "KNOWS": lance.dataset(uri)} diff --git a/python/benchmarks/csr_delta_accumulated.py b/python/benchmarks/csr_delta_accumulated.py index e2dcf73..5dbfac2 100644 --- a/python/benchmarks/csr_delta_accumulated.py +++ b/python/benchmarks/csr_delta_accumulated.py @@ -4,7 +4,7 @@ """Refresh cost as CSR deltas accumulate over many small commits. Uses a graph written by ``csr_expand.py``. On a fresh copy of the edge table, -appends ``--step`` of the edges ``--commits`` times, calling ``refresh()`` after +appends ``--step`` of the edges ``--commits`` times, calling ``checkout_latest()`` after each, and prints each refresh's time with the delta size so far. With deltas that are copied on every refresh, the time grows with the accumulated delta; with shared delta layers it should stay roughly proportional to the step. @@ -59,7 +59,7 @@ def main() -> None: ) lance.write_dataset(new, uri, mode="append") start = time.perf_counter() - engine.refresh() + engine.checkout_latest() refresh_ms = (time.perf_counter() - start) * 1e3 stats = engine.csr_stats()["KNOWS"] print( diff --git a/python/benchmarks/csr_expand.py b/python/benchmarks/csr_expand.py index 6ef4e92..5134b51 100644 --- a/python/benchmarks/csr_expand.py +++ b/python/benchmarks/csr_expand.py @@ -124,7 +124,7 @@ def open_engine(): out[name] = {**check, **summary(samples)} t = time.perf_counter() - engine.refresh() + engine.checkout_latest() out["refresh_unchanged_ms"] = (time.perf_counter() - t) * 1e3 # Recorded before the copy below, which materializes the edge table. out["peak_rss_mib"] = peak_rss_mib() @@ -149,7 +149,7 @@ def open_engine(): ) lance.write_dataset(new_edges, knows, mode="append") t = time.perf_counter() - engine.refresh() + engine.checkout_latest() out["refresh_after_append_ms"] = (time.perf_counter() - t) * 1e3 if mode == "csr": out["csr_rebuild_ms"] = engine.csr_stats()["KNOWS"]["build_ms"] diff --git a/python/benchmarks/engine_cache.py b/python/benchmarks/engine_cache.py index 05a1ff2..ab32a06 100644 --- a/python/benchmarks/engine_cache.py +++ b/python/benchmarks/engine_cache.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""Engine construction, first-query, warm-query and refresh timings. +"""Engine construction, first-query, warm-query and checkout_latest() timings. Runs against either the upstream build (``--module lance_graph``) or this fork (``--module lance_csr``) on the same on-disk Lance data, so the two can be compared @@ -112,11 +112,13 @@ def emit(case, **values): warm.append(t) emit("warm_query", **summary(warm)) - # Upstream has no refresh(); its equivalent is building a new engine. + # Upstream has no checkout_latest() (refresh() in older lance-csr); its equivalent + # is building a new engine. refresh = [] + checkout = getattr(engine, "checkout_latest", None) or getattr(engine, "refresh", None) for _ in range(min(args.repeats, 10)): - if hasattr(engine, "refresh"): - t, _ = timed(engine.refresh) + if checkout is not None: + t, _ = timed(checkout) else: t, engine = timed(lambda: mod.CypherEngine(config, open_datasets())) refresh.append(t) diff --git a/python/python/lance_csr/_internal/graph.pyi b/python/python/lance_csr/_internal/graph.pyi index b636099..64b74f0 100644 --- a/python/python/lance_csr/_internal/graph.pyi +++ b/python/python/lance_csr/_internal/graph.pyi @@ -133,6 +133,7 @@ class CypherEngine: def explain( self, query: str, params: dict[str, _ParameterT] | None = None ) -> str: ... + def checkout_latest(self) -> None: ... def refresh(self) -> None: ... def optimize_csr(self) -> None: ... def delete_stored_csr(self) -> int: ... diff --git a/python/python/lance_csr/cypher.py b/python/python/lance_csr/cypher.py index c8ce1ab..a2b795a 100644 --- a/python/python/lance_csr/cypher.py +++ b/python/python/lance_csr/cypher.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from enum import IntEnum from typing import TYPE_CHECKING, Any, Self, final @@ -47,9 +48,9 @@ class CypherEngine: Lance datasets are read lazily, pinned to the version passed in: later writes, by this or another process, aren't visible until - [`refresh`][lance_csr.CypherEngine.refresh]. Each dataset is pinned separately, so - versions across tables are not a transactional snapshot. PyArrow inputs are held in - memory and never change; create a new engine to replace them. + [`checkout_latest`][lance_csr.CypherEngine.checkout_latest]. Each dataset is pinned + separately, so versions across tables are not a transactional snapshot. PyArrow + inputs are held in memory and never change; create a new engine to replace them. [`execute`][lance_csr.CypherEngine.execute] and [`execute_batches`][lance_csr.CypherEngine.execute_batches] reuse up to 128 @@ -58,7 +59,8 @@ class CypherEngine: call. Container parameters, rich path values, `CALL` subqueries, parameterized `SKIP`/`LIMIT` and unsupported template shapes use ordinary planning. [`explain`][lance_csr.CypherEngine.explain] always plans directly, and a successful - `refresh` clears the cache. See [Repeated queries](lance.md#repeated-queries). + `checkout_latest` clears the cache. See [Repeated + queries](lance.md#repeated-queries). Parameters ---------- @@ -73,17 +75,17 @@ class CypherEngine: endpoints, and node indexes for labels with unique integer keys. target_partitions : int or None Keyword-only. A target from 1 to 1024 for DataFusion query partitioning; `None` - keeps DataFusion's default. `refresh` preserves it. It is not a thread or memory - limit. See [Query parallelism](lance.md#query-parallelism). + keeps DataFusion's default. `checkout_latest` preserves it. It is not a thread + or memory limit. See [Query parallelism](lance.md#query-parallelism). persist_csr : bool Keyword-only, experimental, needs `use_csr=True`. Store each relationship dataset's CSR inside the dataset's own directory, at `_lance_csr/.arrow`, and load it instead of building when an engine starts. Changes since the stored version apply as a delta under the same 25% - threshold as `refresh`; past it, the CSR is rebuilt and stored again. Lance's - `cleanup_old_versions` leaves the folder alone. PyArrow inputs aren't stored, - and a dataset that can't be written (read-only storage or credentials) is - built as without it. See [Storing the CSR](csr.md#storing-the-csr). + threshold as `checkout_latest`; past it, the CSR is rebuilt and stored again. + Lance's `cleanup_old_versions` leaves the folder alone. PyArrow inputs aren't + stored, and a dataset that can't be written (read-only storage or credentials) + is built as without it. See [Storing the CSR](csr.md#storing-the-csr). Raises ------ @@ -200,8 +202,8 @@ def execute_batches[ParameterT: Parameter]( Batches are computed as the reader is consumed. Scans, filters and projections stream with bounded memory; joins, sorts and aggregations may still buffer. The reader keeps its snapshot alive, so a concurrent - [`refresh`][lance_csr.CypherEngine.refresh] doesn't affect it. Planning errors - raise immediately; execution errors raise while reading. + [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] doesn't affect it. + Planning errors raise immediately; execution errors raise while reading. Parameters ---------- @@ -257,36 +259,52 @@ def explain[ParameterT: Parameter]( """ return self._inner.explain(query, params) - def refresh(self) -> None: + def checkout_latest(self) -> None: """Move every Lance dataset to its latest version. - Updates the CSR, node indexes and statistics to match. Datasets whose version - and commit time are unchanged keep their open handle. The replacement is - validated in full before it's swapped in: on error nothing changes and the - previous snapshot stays usable. Queries already running finish on the snapshot - they started with. PyArrow inputs are unchanged. Concurrent calls run one at a - time, and `target_partitions` is preserved. See - [Lance datasets](lance.md#refresh). + The engine's counterpart of Lance's `LanceDataset.checkout_latest()`, for every + dataset at once. Updates the CSR, node indexes and statistics to match. Datasets + whose version and commit time are unchanged keep their open handle. The + replacement is validated in full before it's swapped in: on error nothing + changes and the previous snapshot stays usable. Queries already running finish + on the snapshot they started with. PyArrow inputs are unchanged. Concurrent + calls run one at a time, and `target_partitions` is preserved. See [Lance + datasets](lance.md#checkout-latest). + """ + self._inner.checkout_latest() + + def refresh(self) -> None: + """Deprecated alias of `checkout_latest`. + + Behaves exactly like + [`checkout_latest`][lance_csr.CypherEngine.checkout_latest], and warns with + `DeprecationWarning`. """ - self._inner.refresh() + warnings.warn( + "CypherEngine.refresh() is deprecated; use checkout_latest(), named after " + "Lance's LanceDataset.checkout_latest()", + DeprecationWarning, + stacklevel=2, + ) + self.checkout_latest() def optimize_csr(self) -> None: """Rebuild each CSR that carries a delta (experimental; needs `use_csr=True`). The CSR's counterpart of Lance's `optimize_indices()`. - [`refresh`][lance_csr.CypherEngine.refresh] applies Lance writes as a delta - over the base CSR, and rebuilds on its own only once pending changes pass 25% - of the base. This rebuilds every CSR that carries a delta now, at the version - the engine is pinned to, so lookups stop carrying it. It scans every edge - again, so call it after a bulk load or at a quiet time, not after every - refresh. It doesn't move to - newer versions: call `refresh` first for that. CSRs without a delta, and - PyArrow inputs, are left as they are, and query results never change. - Afterwards [`csr_stats`][lance_csr.CypherEngine.csr_stats] reports `update` - `"rebuild"`, `unindexed_edges` 0 and `base_version` equal to `version`. As for - `refresh`, the replacement is validated in full before it's swapped in, and the - two run one at a time. See - [Rebuilding the CSR](csr.md#rebuilding-the-csr). + [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] applies Lance writes + as a delta over the base CSR, and rebuilds on its own only once pending changes + pass 25% of the base. This rebuilds every CSR that carries a delta now, at the + version the engine is pinned to, so lookups stop carrying it. It scans every + edge again, so call it after a bulk load or at a quiet time, not after every + `checkout_latest`. It doesn't move to newer versions: call `checkout_latest` + first for that. CSRs without a delta, and PyArrow inputs, are left as they are, + and query results never change. Afterwards + [`csr_stats`][lance_csr.CypherEngine.csr_stats] reports `update` `"rebuild"`, + `unindexed_edges` 0 and `base_version` equal to `version`. As for + `checkout_latest`, the replacement is validated in full before it's swapped in, + and the two run one at a time. See [Rebuilding the + CSR](csr.md#rebuilding-the-csr). """ self._inner.optimize_csr() @@ -298,7 +316,7 @@ def delete_stored_csr(self) -> int: `persist_csr`. The engine keeps the CSR it already holds in memory, so queries aren't affected. To go back to in-memory builds only, create later engines without `persist_csr`: an engine with it stores a new file at its next full - build. Runs one at a time with `refresh` and `optimize_csr`. See + build. Runs one at a time with `checkout_latest` and `optimize_csr`. See [Storing the CSR](csr.md#storing-the-csr). Returns @@ -325,9 +343,9 @@ def node_index_stats(self) -> dict[str, NodeIndexStats]: def csr_stats(self) -> dict[str, CsrStats]: """Statistics for each CSR adjacency index (experimental; needs `use_csr=True`). - An identical result after [`refresh`][lance_csr.CypherEngine.refresh] means the - snapshot itself didn't change. See - [Inspecting the CSR](csr.md#inspecting-the-csr). + An identical result after + [`checkout_latest`][lance_csr.CypherEngine.checkout_latest] means the snapshot + itself didn't change. See [Inspecting the CSR](csr.md#inspecting-the-csr). Returns ------- @@ -339,7 +357,7 @@ def csr_stats(self) -> dict[str, CsrStats]: full build, held outside the base CSR. - `unindexed_edges`: live edges outside the base CSR, appended since its last full build and not since removed. - - `update`: how the last refresh or + - `update`: how the last build, `checkout_latest` or [`optimize_csr`][lance_csr.CypherEngine.optimize_csr] produced it: `"build"` (first), `"rebuild"` (full), `"loaded"` (from a file stored with `persist_csr=True`, with any changes since applied as a delta), `"delta"` diff --git a/python/python/tests/test_call_subqueries.py b/python/python/tests/test_call_subqueries.py index 9d4a498..454dfe8 100644 --- a/python/python/tests/test_call_subqueries.py +++ b/python/python/tests/test_call_subqueries.py @@ -413,7 +413,7 @@ def test_rich_path_reader_retains_snapshot_after_refresh(graph, tmp_path): lance.write_dataset( pa.table({"src": [20], "dst": [30]}), paths["KNOWS"], mode="append" ) - engine.refresh() + engine.checkout_latest() assert engine.execute(query, {"start": 20}).column(0).to_pylist() == [ [20, 30], [20, 40], diff --git a/python/python/tests/test_engine_csr.py b/python/python/tests/test_engine_csr.py index 28aa28e..edb3c59 100644 --- a/python/python/tests/test_engine_csr.py +++ b/python/python/tests/test_engine_csr.py @@ -285,7 +285,7 @@ def test_csr_tracks_edge_changes_but_not_node_changes(tmp_path): tmp_path / "person.lance", mode="append", ) - engine.refresh() + engine.checkout_latest() assert engine.csr_stats() == before new_edge = pa.table( @@ -297,7 +297,7 @@ def test_csr_tracks_edge_changes_but_not_node_changes(tmp_path): ) lance.write_dataset(new_edge, tmp_path / "knows.lance", mode="append") assert engine.execute(query).num_rows == 0 # pinned snapshot - engine.refresh() + engine.checkout_latest() after = engine.csr_stats() assert after["KNOWS"]["version"] == 2 and after["KNOWS"]["edges"] == 8 assert engine.execute(query).column(0).to_pylist() == [1] @@ -337,7 +337,7 @@ def test_high_degree_stream_keeps_its_snapshot_across_deletions(tmp_path): assert 0 < first.num_rows < n # A live reader retains the old adjacency, including both copies of 0->1. knows.delete("dst = 1") - engine.refresh() + engine.checkout_latest() assert engine.csr_stats()["KNOWS"]["tombstones"] == 2 counts = {} for batch in chain([first], reader): @@ -429,7 +429,7 @@ def test_node_index_rebuilt_only_when_node_version_changes(tmp_path): } ) lance.write_dataset(edge, tmp_path / "knows.lance", mode="append") - engine.refresh() + engine.checkout_latest() assert engine.node_index_stats() == before query = "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 6 RETURN b.name" assert engine.execute(query).num_rows == 0 # node 7 does not exist yet @@ -439,7 +439,7 @@ def test_node_index_rebuilt_only_when_node_version_changes(tmp_path): tmp_path / "person.lance", mode="append", ) - engine.refresh() + engine.checkout_latest() after = engine.node_index_stats() assert after["Person"]["version"] == 2 and after["Person"]["rows"] == 7 assert engine.execute(query).column(0).to_pylist() == ["g"] diff --git a/python/python/tests/test_engine_csr_delta.py b/python/python/tests/test_engine_csr_delta.py index 6c928eb..345b222 100644 --- a/python/python/tests/test_engine_csr_delta.py +++ b/python/python/tests/test_engine_csr_delta.py @@ -3,7 +3,7 @@ """Incremental CSR maintenance: edge commits applied as deltas and tombstones. -After each change and ``refresh()``, the CSR engine must return exactly what a +After each change and ``checkout_latest()``, the CSR engine must return exactly what a fresh join-path engine and a freshly built CSR engine return on the new data. """ @@ -68,7 +68,7 @@ def make(**write_kwargs): def check(engine, knows_uri): - """CSR engine (after refresh) == fresh join engine == fresh CSR engine.""" + """CSR engine (after checkout_latest) == fresh join engine == fresh CSR engine.""" datasets = { "Person": lance.dataset(knows_uri.parent / "person.lance"), "KNOWS": lance.dataset(knows_uri), @@ -95,7 +95,7 @@ def test_initial_build_reports_build(graph): def test_append_applied_as_delta(graph): engine, uri = graph() lance.write_dataset(edges([(1, 2), (39, 0), (5, 5)], 1000), uri, mode="append") - engine.refresh() + engine.checkout_latest() s = stats(engine) assert (s["update"], s["delta_edges"], s["tombstones"]) == ("delta", 3, 0) assert s["edges"] == 203 and s["unindexed_edges"] == 3 @@ -106,7 +106,7 @@ def test_append_applied_as_delta(graph): def test_delete_applied_as_tombstones(graph): engine, uri = graph() lance.dataset(uri).delete("since % 9 = 0") - engine.refresh() + engine.checkout_latest() s = stats(engine) assert s["update"] == "delta" and s["tombstones"] == 23 and s["delta_edges"] == 0 check(engine, uri) @@ -116,7 +116,7 @@ def test_delete_one_of_duplicate_edges(graph): engine, uri = graph() # since 190..199 are the duplicated i -> i+1 edges; delete one copy of 2 -> 3. lance.dataset(uri).delete("since = 192") - engine.refresh() + engine.checkout_latest() query = "MATCH (a:Person)-[:KNOWS]->(b:Person) WHERE a.id = 2 RETURN b.id" expected = sorted(d for s, d in BASE if s == 2) expected.remove(3) @@ -127,7 +127,7 @@ def test_delete_one_of_duplicate_edges(graph): def test_delete_whole_fragment(graph): engine, uri = graph() lance.dataset(uri).delete("since >= 40 AND since < 80") - engine.refresh() + engine.checkout_latest() assert stats(engine)["tombstones"] == 40 check(engine, uri) @@ -135,7 +135,7 @@ def test_delete_whole_fragment(graph): def test_update_endpoint_values(graph): engine, uri = graph() lance.dataset(uri).update({"dst": "0"}, where="since < 15") - engine.refresh() + engine.checkout_latest() assert stats(engine)["update"] == "delta" check(engine, uri) @@ -144,7 +144,7 @@ def test_non_endpoint_column_change_reuses_csr(graph): engine, uri = graph() before = stats(engine) lance.dataset(uri).add_columns({"weight": "since * 2"}) - engine.refresh() + engine.checkout_latest() after = stats(engine) assert after["update"] == "reused" and after["edges"] == before["edges"] assert after["version"] > before["version"] @@ -154,11 +154,11 @@ def test_non_endpoint_column_change_reuses_csr(graph): def test_sequential_deltas_accumulate(graph): engine, uri = graph() lance.write_dataset(edges([(7, 8), (7, 9)], 2000), uri, mode="append") - engine.refresh() + engine.checkout_latest() lance.dataset(uri).delete("since = 2000 OR since = 0") - engine.refresh() + engine.checkout_latest() lance.write_dataset(edges([(7, 8)], 3000), uri, mode="append") - engine.refresh() + engine.checkout_latest() s = stats(engine) assert s["update"] == "delta" and s["delta_edges"] == 3 and s["tombstones"] == 2 # One tombstone removes an appended edge, so two of three appends are still live. @@ -170,7 +170,7 @@ def test_optimize_folds_delta_into_base(graph): engine, uri = graph() lance.write_dataset(edges([(7, 8), (7, 9)], 2000), uri, mode="append") lance.dataset(uri).delete("since = 2000 OR since = 0") - engine.refresh() + engine.checkout_latest() before = stats(engine) assert before["update"] == "delta" and before["unindexed_edges"] == 1 @@ -210,7 +210,7 @@ def test_compaction_keeps_results(graph): ds = lance.dataset(uri) ds.delete("since < 30") ds.optimize.compact_files(target_rows_per_fragment=1000) - engine.refresh() + engine.checkout_latest() check(engine, uri) @@ -219,14 +219,14 @@ def test_deltas_survive_cleanup_of_old_versions(graph): engine, uri = graph() lance.write_dataset(edges([(3, 4), (4, 3)], 4000), uri, mode="append") lance.write_dataset(edges([(5, 6), (6, 5)], 4002), uri, mode="append") - engine.refresh() + engine.checkout_latest() # Compaction rewrites the two small fragments; cleanup deletes their files. lance.dataset(uri).optimize.compact_files(target_rows_per_fragment=40) lance.dataset(uri).delete("since = 0 OR since = 4003") lance.dataset(uri).cleanup_old_versions( older_than=timedelta(0), delete_unverified=True ) - engine.refresh() + engine.checkout_latest() s = stats(engine) assert (s["update"], s["delta_edges"], s["tombstones"]) == ("delta", 7, 5) check(engine, uri) @@ -235,7 +235,7 @@ def test_deltas_survive_cleanup_of_old_versions(graph): def test_non_endpoint_column_change_is_visible_through_relationship(graph): engine, uri = graph() lance.dataset(uri).add_columns({"weight": "since * 2"}) - engine.refresh() + engine.checkout_latest() assert stats(engine)["update"] == "reused" got = engine.execute( "MATCH (a:Person)-[r:KNOWS]->(b:Person) RETURN r.since, r.weight" @@ -247,7 +247,7 @@ def test_non_endpoint_column_change_is_visible_through_relationship(graph): def test_overwrite_triggers_rebuild(graph): engine, uri = graph() lance.write_dataset(edges([(1, 2), (2, 3)]), uri, mode="overwrite") - engine.refresh() + engine.checkout_latest() s = stats(engine) assert (s["update"], s["edges"], s["delta_edges"]) == ("rebuild", 2, 0) check(engine, uri) @@ -257,7 +257,7 @@ def test_large_delta_triggers_rebuild(graph): engine, uri = graph() many = [(i % N, (i + 3) % N) for i in range(100)] # 50% of the base lance.write_dataset(edges(many, 5000), uri, mode="append") - engine.refresh() + engine.checkout_latest() s = stats(engine) assert (s["update"], s["edges"], s["delta_edges"]) == ("rebuild", 300, 0) assert s["unindexed_edges"] == 0 and s["base_version"] == s["version"] @@ -269,6 +269,6 @@ def test_stable_row_ids_with_deletes_and_appends(graph): ds = lance.dataset(uri) ds.delete("since % 5 = 1") lance.write_dataset(edges([(0, 39)], 9000), uri, mode="append") - engine.refresh() + engine.checkout_latest() assert stats(engine)["update"] == "delta" check(engine, uri) diff --git a/python/python/tests/test_engine_lance.py b/python/python/tests/test_engine_lance.py index 0c81356..7c14962 100644 --- a/python/python/tests/test_engine_lance.py +++ b/python/python/tests/test_engine_lance.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""CypherEngine over on-disk Lance datasets: snapshots, refresh and streaming. +"""CypherEngine over on-disk Lance datasets: snapshots, checkout_latest() and streaming. The engine pins the dataset version it was given; writes become visible only -after ``refresh()``. Expected values are computed independently of the engine. +after ``checkout_latest()``. Expected values are computed independently of the engine. """ import re @@ -109,16 +109,25 @@ def test_snapshot_until_refresh_after_append(graph): lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append") assert count_people(engine) == N - engine.refresh() + engine.checkout_latest() assert count_people(engine) == N + 5 assert engine.dataset_versions()["Person"] == 2 +def test_refresh_is_a_deprecated_alias_of_checkout_latest(graph): + path, person, knows = graph + engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) + lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append") + with pytest.warns(DeprecationWarning, match="checkout_latest"): + engine.refresh() + assert count_people(engine) == N + 5 + + def test_refresh_without_changes_keeps_results(graph): _, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) before = engine.execute("MATCH (p:Person) WHERE p.id < 3 RETURN p.name") - engine.refresh() + engine.checkout_latest() after = engine.execute("MATCH (p:Person) WHERE p.id < 3 RETURN p.name") assert before.equals(after) assert engine.dataset_versions() == {"Person": 1, "KNOWS": 1} @@ -136,7 +145,7 @@ def test_refresh_sees_overwrite_with_added_and_retyped_columns(graph): lance.write_dataset(replacement, path / "person.lance", mode="overwrite") assert count_people(engine) == N # still pinned - engine.refresh() + engine.checkout_latest() out = engine.execute("MATCH (p:Person) RETURN p.name, p.age") assert out.column(0).type == pa.int64() assert sorted(out.column(1).to_pylist()) == [30, 40] @@ -149,7 +158,7 @@ def test_refresh_sees_table_replaced_at_same_location(graph): # Delete and recreate: the new dataset is version 1 again, like the old one. shutil.rmtree(path / "person.lance") lance.write_dataset(people(0, 3), path / "person.lance") - engine.refresh() + engine.checkout_latest() assert count_people(engine) == 3 @@ -163,7 +172,7 @@ def test_failed_refresh_keeps_previous_snapshot(graph, use_csr): pa.table({"name": ["x"]}), path / "person.lance", mode="overwrite" ) with pytest.raises(ValueError, match="id"): - engine.refresh() + engine.checkout_latest() assert count_people(engine) == N assert engine.dataset_versions()["Person"] == 1 @@ -185,8 +194,8 @@ def test_historical_version_is_pinned_and_engines_independent(graph): assert count_people(old) == N assert count_people(new) == N + 5 - # refresh() moves a historical snapshot to the latest version. - old.refresh() + # checkout_latest() moves a historical snapshot to the latest version. + old.checkout_latest() assert count_people(old) == N + 5 @@ -201,7 +210,7 @@ class Wrapped(lance.LanceDataset): # Treated as Lance (pinned, refreshable), not materialized as Arrow. assert engine.dataset_versions()["Person"] == 1 lance.write_dataset(people(N, N + 1), path / "person.lance", mode="append") - engine.refresh() + engine.checkout_latest() assert count_people(engine) == N + 1 @@ -224,7 +233,7 @@ def reader(): lance.write_dataset( people(N + 5 * (i - 1), N + 5 * i), path / "person.lance", mode="append" ) - engine.refresh() + engine.checkout_latest() for t in threads: t.join() @@ -264,7 +273,7 @@ def test_execute_batches_keeps_snapshot_across_refresh(graph, use_csr): first = reader.read_next_batch() lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append") - engine.refresh() + engine.checkout_latest() remaining = reader.read_all() assert first.num_rows + remaining.num_rows == N diff --git a/python/python/tests/test_path_values.py b/python/python/tests/test_path_values.py index 6a4baf4..c21e391 100644 --- a/python/python/tests/test_path_values.py +++ b/python/python/tests/test_path_values.py @@ -274,7 +274,7 @@ def test_path_inventory_keeps_lance_snapshot(tmp_path): lance.write_dataset( pa.table({"src": [1], "dst": [2], "weight": [7]}), edges_path, mode="overwrite" ) - engine.refresh() + engine.checkout_latest() assert reader.read_all().to_pylist() == [ {"names": ["old", "target"], "weights": [5]} ] @@ -599,7 +599,7 @@ def test_cross_label_shortest_inventory_isolates_refresh(cross_label_graph, tmp_ lance.write_dataset( pa.table({"src": [1], "dst": [2]}), paths["LIVES"], mode="overwrite" ) - engine.refresh() + engine.checkout_latest() assert reader.read_all().to_pylist() == [{"ids": [1, 1]}] for start in [1, 2, 1]: assert engine.execute(query, {"start": start}).to_pylist() == ( diff --git a/python/python/tests/test_target_partitions.py b/python/python/tests/test_target_partitions.py index 2bc0f1e..676ad61 100644 --- a/python/python/tests/test_target_partitions.py +++ b/python/python/tests/test_target_partitions.py @@ -58,7 +58,7 @@ def test_partition_targets_preserve_queries_and_refresh( lance.write_dataset( pa.table({"id": [5]}), tmp_path / "Person.lance", mode="append" ) - engine.refresh() + engine.checkout_latest() assert reader.read_all().to_pylist() == [{"id": value} for value in range(1, 5)] count = 5 if storage == "lance" else 4 assert engine.execute(scan).to_pylist() == [ diff --git a/python/python/tests/test_typing.py b/python/python/tests/test_typing.py index 87b21cc..21378ea 100644 --- a/python/python/tests/test_typing.py +++ b/python/python/tests/test_typing.py @@ -190,7 +190,7 @@ def test_stub_check_detects_missing_method(): node.body = [ item for item in node.body - if not isinstance(item, ast.FunctionDef) or item.name != "refresh" + if not isinstance(item, ast.FunctionDef) or item.name != "checkout_latest" ] with pytest.raises(AssertionError, match="public members differ"): check_class(node, _internal.graph.CypherEngine) diff --git a/python/python/tests/typing/public_api.py b/python/python/tests/typing/public_api.py index 539614f..b2b3c36 100644 --- a/python/python/tests/typing/public_api.py +++ b/python/python/tests/typing/public_api.py @@ -54,7 +54,7 @@ def public_api( assert_type(engine.execute(query, params), Table) assert_type(engine.execute_batches(query, params), RecordBatchReader) assert_type(engine.explain(query, params), str) - assert_type(engine.refresh(), None) + assert_type(engine.checkout_latest(), None) assert_type(engine.config(), GraphConfig) assert_type(engine.dataset_versions(), dict[str, int]) assert_type(engine.node_index_stats()["person"]["rows"], int) From 706e1810fe10a3cee5fe0ddc2571fbb6812080fe Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 2 Nov 2026 15:00:17 -0500 Subject: [PATCH 9/9] docs(csr): rewrite keeping the csr current for newcomers Explain what checkout_latest() and optimize_csr() do and why before how, drop the remaining refresh wording, and add the what-and-why-first rule to the docs style guide. --- docs/AGENTS.md | 10 +++- docs/csr.md | 159 ++++++++++++++++++++----------------------------- docs/lance.md | 5 +- 3 files changed, 73 insertions(+), 101 deletions(-) diff --git a/docs/AGENTS.md b/docs/AGENTS.md index cd1e3df..13327bf 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -76,8 +76,14 @@ them. ## Style -- Lead with what the reader wants to do, then show it. Mix short prose with code; a concept that takes more - than two paragraphs probably needs an example or a diagram. +- Explain what something is and why it exists before how to use it. Don't open a page or section with an + instruction ("Call it after…", "Use this when…"): the reader needs the what and the why to judge the how. Mix + short prose with code; a concept that takes more than two paragraphs probably needs an example or a diagram. +- Don't assume the reader knows the background. Give the context a newcomer needs (for example, that Lance + datasets gain new versions as they're written, so the engine has to be told about them), and define a term, + such as the CSR's delta, before relying on it. +- Say each thing once. A section shouldn't restate what the page already said above it; keep sentences short and + cut words that add no information. - Use Zensical components where they help the reader, not for decoration: - admonitions (`!!! tip`, `note`, `warning`, `danger`, `info`, `example`, `failure`, `bug`) for advice, caveats and known issues; `???` makes a collapsible one for long detail; diff --git a/docs/csr.md b/docs/csr.md index d6e0c80..f37a788 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -290,47 +290,18 @@ print(sorted(engine.node_index_stats())) # ['Person'] ## Keeping the CSR current -A Lance dataset can change at any time. You, another process or LanceDB can append rows, delete them or compact -the files, and every write commits a new version of the dataset. The engine doesn't follow those changes as they -happen: when you create it, it opens each Lance dataset at its current version, builds the CSR from that version, -and keeps answering queries from that snapshot until you tell it to look again. That keeps queries consistent with -each other, but it also means the CSR falls behind as soon as anyone writes to the data. - -Two calls bring it back up to date, and they do different jobs: - -1. **[`checkout_latest()`](#checkout-latest)** moves the engine to the newest version of each Lance dataset. Instead of rebuilding - the CSR, it records the edges added or removed since the last build as a *delta*, a small set of changes kept - next to the CSR. Call it whenever your queries should see new writes. -2. **[`optimize_csr()`](#rebuilding-the-csr)** rebuilds a CSR from scratch, so it no longer needs its delta. Call - it occasionally, once enough changes have built up that you'd rather lookups stopped working through them. - -The two calls aren't a matched pair, though. `checkout_latest()` isn't really a CSR operation: it's named after, and -works like, Lance's own `LanceDataset.checkout_latest()`, and it moves the whole engine to the newest version of every -Lance dataset it reads. Node tables, node indexes and the statistics used to plan joins all move with it, and keeping -the CSR in step is just one part of that, done as cheaply as possible with a delta. `optimize_csr()` is the call that's -about the CSR itself: it rebuilds a CSR in a running engine when you decide, rather than waiting for the delta to get -too large. It plays the part Lance's `optimize_indices()` plays for Lance's own indexes. - -If you gave the engine in-memory PyArrow tables instead of Lance datasets, neither call applies: those tables can't -change underneath the engine, so their CSR is never out of date. To query new in-memory data, create a new engine. - -### `checkout_latest()`: pick up new writes { #checkout-latest } - -Because each Lance dataset can evolve independently as you write to it, the engine has to be told that a newer -version is available. `checkout_latest()` does that: it checks every Lance dataset for a newer version and moves the engine to -it. Call it after the writes your queries should see, whether they came from `pylance`, LanceDB or another process. -If one writer updates several tables, wait until all of them have committed before you call it, since each Lance -dataset is pinned on its own (see [snapshots are per Lance dataset](lance.md#lazy-version-pinned-reads)). When nothing -has changed, `checkout_latest()` keeps what the engine already has and returns quickly, so calling it on a timer or before -each batch of queries is fine. - -Rebuilding the whole CSR on every call would mean scanning every edge again, even when only a handful changed. -So `checkout_latest()` reads only the new parts of each relationship dataset and applies the changes on top of the CSR from -the last full build, which the rest of this page calls the *base*. Appended edges go into the delta, and deleted -edges are marked so that lookups skip them; `csr_stats()` counts those marks as `tombstones`. Answers are exactly -what a full rebuild would give. To see what a call did, check `update` in [`csr_stats()`](#csr_stats): `reused` -means nothing changed, `delta` means the changes were applied, and `rebuild` means the CSR was built again from -scratch, which happens in the cases below. +A Lance dataset gets a new version every time someone writes to it, whether that's appending rows, deleting them +or compacting its files. The engine doesn't switch versions on its own. It answers every query from the versions it +opened, so that queries stay consistent with each other, which means new writes stay invisible until you ask for +them. + +`checkout_latest()` is how you ask. It moves the engine to the newest version of every Lance dataset it reads, the +way Lance's own `checkout_latest()` does for a single dataset (see Lance's +[time travel](https://lance.org/quickstart/#time-travel)), and it updates everything the engine built from those +datasets, the CSR included. Afterwards, queries see the new data and their answers are exact. When nothing has +changed, it returns almost immediately, so it's safe to call often, even on a timer. If a writer updates several +tables, call it once they've all committed, since each Lance dataset is +[pinned on its own](lance.md#lazy-version-pinned-reads). === "Python" @@ -361,80 +332,76 @@ scratch, which happens in the cases below. --8<-- "crates/lance-csr/examples/embedded_query.rs:checkout_latest" ``` -??? info "How `checkout_latest()` decides" +!!! caution "In-memory PyArrow tables" + + If you gave the engine in-memory PyArrow tables instead of Lance datasets, there's nothing to update: those + tables can't change underneath the engine, so their CSR is never out of date. To query new in-memory data, + create a new engine. + +### How the CSR keeps up with new writes { #checkout-latest } + +Rebuilding a CSR means reading every edge again, which takes about a second for 10 million edges. Most writes only +add or remove a few edges, so `checkout_latest()` doesn't rebuild the CSR every time. It reads just the new data, +keeps the added edges in a short list next to the CSR, called the *delta*, and marks deleted edges so that lookups +skip them. Every lookup checks the delta as well as the CSR, so the new edges show up in answers straight away. + +The delta can't grow forever. Once it reaches 25% of the CSR's size, `checkout_latest()` rebuilds the CSR +from scratch instead, which empties the delta. You don't have to manage any of this. To see what a call did, look +at `update` in [`csr_stats()`](#csr_stats): `reused` (nothing changed), `delta` (new edges went into the delta) or +`rebuild` (the CSR was rebuilt), as in the example above. + +??? info "Details: when `checkout_latest()` rebuilds" ```mermaid flowchart LR R["checkout_latest()"] -- "no edge changes" --> U["reused"] - R -- "changes under 25% of the base" --> DL["delta"] - R -- "25% or more, or new endpoint columns" --> RB["rebuild"] + R -- "delta under 25% of the CSR" --> DL["delta"] + R -- "25% or more, overwritten, or new endpoint columns" --> RB["rebuild"] ``` - - `checkout_latest()` rebuilds a CSR instead of adding to its delta once the changes since the last full build would reach - 25% of the base, or when the relationship dataset was overwritten or its endpoint columns changed. The - threshold is fixed: every lookup reads the delta as well as the base, so a large delta slows lookups down, and - a rebuild clears it. `python/benchmarks/csr_delta_accumulated.py` measures that trade-off. - - Deleted and rewritten rows are recognised by the edge row that each CSR entry records, so compacting a dataset - or running `cleanup_old_versions()` doesn't force a rebuild: nothing needs to be read from the old version. - - Writes to a node table never touch the CSR, because the CSR stores the target nodes' keys rather than their row - positions. Only that label's node index is rebuilt. - - The new CSR is built in full before the engine switches to it. If anything fails along the way, nothing - changes, and queries that were already running finish on the snapshot they started with. - -### `optimize_csr()`: rebuild to clear the delta { #rebuilding-the-csr } - -The delta keeps `checkout_latest()` cheap, but every lookup has to check it as well as the base. It only rebuilds once -the delta reaches 25% of the base, so a smaller delta can stay around indefinitely and slow lookups a little the -whole time. `optimize_csr()` lets you clear it at a moment you choose. It rebuilds every CSR that has a delta, from a -scan of all its edges, at the version the engine is already pinned to, and leaves CSRs without a delta alone. That -costs about as much as building the CSR the first time, so it's best kept for after a bulk load, before a run of -latency-sensitive queries, or a quiet period, rather than called after every `checkout_latest()`. Query results are the same -before and after. + - Compaction and `cleanup_old_versions()` don't force a rebuild: deleted rows are matched by the edge row each + CSR entry records. + - Writes to a node table never touch the CSR, which stores node keys rather than row positions. Only that + label's node index is rebuilt. + +### Optional: rebuilding early with `optimize_csr()` { #rebuilding-the-csr } + +Most applications never need this method. `checkout_latest()` only rebuilds the CSR once the delta reaches 25% +of the CSR's size, and until then the delta stays in place. In some situations, however, it's worth rebuilding +sooner than that: + +- **Large traversals.** Checking the delta on every lookup makes big traversals a little slower (in one of our experiments, we measured 5-13% slower + with a delta of 20% of the CSR, on a graph with 10 million edges). Point lookups aren't affected. +- **Stored CSRs.** With [`persist_csr=True`](#storing-the-csr), the stored file holds the CSR without its delta, so + every new engine re-applies the delta when it starts. A rebuild stores a fresh file, so later starts are fast + again. + +`optimize_csr()` rebuilds every CSR that has a delta, right away, instead of waiting for `checkout_latest()` to do +it. A rebuild takes about as long as building the CSR the first time, and it uses the versions the engine already +has, so new writes need a `checkout_latest()` first: ```python -engine.checkout_latest() # applies the new edges as a delta -engine.optimize_csr() # rebuilds the CSR, now including those edges +engine.checkout_latest() # new edges go into the delta +engine.optimize_csr() # rebuilds the CSR with those edges included stats = engine.csr_stats()["KNOWS"] print(stats["update"], stats["unindexed_edges"]) # rebuild 0 ``` -Afterwards, `csr_stats()` shows `delta_edges`, `tombstones` and `unindexed_edges` at 0, and `base_version` matches -`version`. `optimize_csr()` doesn't look for newer data, so call `checkout_latest()` first if the rebuild should include the -latest writes. - -??? info "How this maps onto Lance's indexes" - - The CSR is maintained the same way Lance maintains its own indexes. A Lance index covers the fragments it was - built from, and data written after that stays unindexed: searches combine the index's results with a scan of the - unindexed rows, so answers stay correct while latency slowly grows, until you call `optimize_indices()` (see - Lance's - [index maintenance guide](https://lance.org/guide/indexing/#maintain-indexes-after-appends) and the - [index format](https://lance.org/format/index/#basic-concepts)). The CSR maps onto that one to one: - - | Lance index | CSR | - | --- | --- | - | The fragments an index was built from | The base CSR, at `base_version` | - | Unindexed rows, scanned alongside the index | The delta (appended edges and tombstones), read by every lookup | - | `num_unindexed_rows` in index statistics | `unindexed_edges` in `csr_stats()` | - | `optimize_indices()` | [`optimize_csr()`][lance_csr.CypherEngine.optimize_csr] | +It's modelled on Lance's `optimize_indices()`, which brings Lance's own indexes up to date after new writes in the +same way (see Lance's [indexing docs](https://lance.org/guide/indexing/#maintain-indexes-after-appends)). ### Which one to call !!! tip "`checkout_latest()` or `optimize_csr()`?" - In short, `checkout_latest()` keeps the engine up to date at little cost, and `optimize_csr()` now and then clears the - delta that `checkout_latest()` leaves behind. - | | `checkout_latest()` | `optimize_csr()` | | --- | --- | --- | - | Data | Moves to the latest version of each Lance dataset | Stays at the version the engine is pinned to | - | Usual effect on a CSR | Applies the new writes as a delta, reading only new fragments | Rebuilds it from a scan of all its edges | - | Rebuilds a CSR when | Its delta would pass 25% of the base | It carries any delta at all | - | Cost | In proportion to the new writes | Like the first build | - | When to call it | Whenever queries should see new writes | Now and then, after `checkout_latest()`, once a delta has built up | - - Whenever either call rebuilds a CSR, it runs the same full build and ends up with the same CSR. Both build the - replacement completely before the engine switches to it, and they never run at the same time. + | What it's for | Showing new writes to your queries | Rebuilding the CSR now instead of later | + | When you need it | Every time your data changes | Rarely: for large traversals with a big delta, or with `persist_csr` | + | Cost | Usually milliseconds; occasionally a full rebuild | About as long as the first build | + + A rebuild gives the same CSR whichever method does it. The two never run at the same time: if you call one + while the other is running, the second waits for the first to finish, and queries keep running meanwhile. ## Storing the CSR diff --git a/docs/lance.md b/docs/lance.md index 9c1c698..dbe8fe6 100644 --- a/docs/lance.md +++ b/docs/lance.md @@ -52,9 +52,8 @@ stay invisible until you call `checkout_latest()`. That gives every query a stab `checkout_latest()` moves every Lance dataset to its latest version. It's named after Lance's own `LanceDataset.checkout_latest()`, which does the same for a single dataset, and it updates everything the engine -built from that data along with it: node indexes, join statistics and, with `use_csr=True`, the CSR. It isn't a -CSR-specific call; to rebuild a CSR, use [`optimize_csr()`](csr.md#rebuilding-the-csr) (see -[Keeping the CSR current](csr.md#keeping-the-csr-current)): +built from that data along with it: node indexes, join statistics and, with `use_csr=True`, the CSR (see +[Keeping the CSR current](csr.md#keeping-the-csr-current) for how the CSR keeps up): ```python lance.write_dataset(