Skip to content

feat: configure WAL merge indexes with key-only ZoneMap deletes #300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
beinan merged 2 commits into from
Oct 3, 2026
Merged

feat: configure WAL merge indexes with key-only ZoneMap deletes #300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02969b3
feat: configure WAL key indexes with bounded ZoneMap predicate deletes
Oct 3, 2026
6ebe7b2
docs: distinguish predicate scan plans in the merge benchmark
Oct 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/lance-context-core/src/datagen_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ pub struct DatagenStoreOptions {
/// reaches the budget, or the generation-count cap, whichever comes first.
/// A generation is indivisible, so even an oversized one is fully merged.
pub merge_max_bytes: Option<usize>,
/// Index used for base-table key lookup during WAL merge.
pub key_index_type: crate::KeyIndexType,
/// Warn when one MemWAL shard has at least this many flushed generations
/// pending merge, sampled on every read. `None` uses the crate default
/// (256); `Some(0)` disables the warn. The `rollout_wal_pending_generations`
Expand Down Expand Up @@ -115,6 +117,7 @@ impl DatagenStore {
merge_after_generations: options.merge_after_generations,
merge_max_generations: options.merge_max_generations,
merge_max_bytes: options.merge_max_bytes,
key_index_type: options.key_index_type,
pending_generations_warn: options.pending_generations_warn,
pending_generations_max: options.pending_generations_max,
merge_budget: options.merge_budget.clone(),
Expand Down
4 changes: 4 additions & 0 deletions crates/lance-context-core/src/generic_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ pub struct GenericStoreOptions {
/// reaches the budget, or the generation-count cap, whichever comes first.
/// A generation is indivisible, so even an oversized one is fully merged.
pub merge_max_bytes: Option<usize>,
/// Index used for base-table key lookup during WAL merge.
pub key_index_type: crate::KeyIndexType,
/// Warn when one MemWAL shard has at least this many flushed generations
/// pending merge, sampled on every read. Every pending generation is a
/// separate dataset a read must open, so this is the read-amplification
Expand Down Expand Up @@ -187,6 +189,7 @@ impl GenericStore {
merge_after_generations: options.merge_after_generations,
merge_max_generations: options.merge_max_generations,
merge_max_bytes: options.merge_max_bytes,
key_index_type: options.key_index_type,
pending_generations_warn: options.pending_generations_warn,
pending_generations_max: options.pending_generations_max,
merge_budget: options.merge_budget.clone(),
Expand Down Expand Up @@ -868,6 +871,7 @@ mod tests {
seal_on_add: true,
shard_id: Some(shard.to_string()),
merge_max_bytes: Some(1024 * 1024),
key_index_type: Default::default(),
merge_budget: Some(budget.clone()),
..Default::default()
};
Expand Down
20 changes: 20 additions & 0 deletions crates/lance-context-core/src/key_index.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/// Base-table key index policy used by WAL merge and scheduled indexing.
/// All writers of a table should use the same policy to avoid rebuilding the
/// index back and forth during maintenance. Opening a store does not rebuild it.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, clap::ValueEnum)]
pub enum KeyIndexType {
/// Exact key lookup; preserves the existing default.
#[default]
Btree,
/// Compact zone summaries, with key-only predicate deletion during merge.
Zonemap,
}

impl KeyIndexType {
pub fn as_str(self) -> &'static str {
match self {
Self::Btree => "btree",
Self::Zonemap => "zonemap",
}
}
}
2 changes: 2 additions & 0 deletions crates/lance-context-core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ mod export;
pub mod generic_codec;
mod generic_store;
mod id;
mod key_index;
pub mod merge_budget;
pub mod merge_write_scope;
pub mod metrics;
Expand Down Expand Up @@ -59,6 +60,7 @@ pub use export::{
pub use generic_codec::{batch_to_rows, ids_from_batch, rows_to_batch, Row};
pub use generic_store::{GenericStore, GenericStoreOptions};
pub use id::{generate_id, new_uuid};
pub use key_index::KeyIndexType;
pub use merge_budget::{MergeMemoryBudget, MergeReservation};
pub use namespace::{ContextNamespace, PartitionInfo, PartitionSelector, PartitionSpec};
pub use record::{
Expand Down
118 changes: 109 additions & 9 deletions crates/lance-context-core/src/rollout_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,8 @@ pub struct RolloutStoreOptions {
/// reaches the budget, or the generation-count cap, whichever comes first.
/// A generation is indivisible, so even an oversized one is fully merged.
pub merge_max_bytes: Option<usize>,
/// Index used for base-table key lookup during WAL merge.
pub key_index_type: crate::KeyIndexType,
/// Warn when one MemWAL shard has at least this many flushed generations
/// pending merge, sampled on every read. `None` uses the crate default
/// (256); `Some(0)` disables the warn. The `rollout_wal_pending_generations`
Expand Down Expand Up @@ -483,6 +485,7 @@ impl RolloutStore {
merge_after_generations,
merge_max_generations,
merge_max_bytes,
key_index_type,
pending_generations_warn,
pending_generations_max,
merge_budget,
Expand All @@ -496,6 +499,7 @@ impl RolloutStore {
merge_after_generations,
merge_max_generations,
merge_max_bytes,
key_index_type,
pending_generations_warn,
pending_generations_max,
merge_budget,
Expand Down Expand Up @@ -712,8 +716,12 @@ impl RolloutStore {
self.base.repair_missing_fragments().await
}

/// Build a ZoneMap scalar index on the base table's `id` column. Idempotent.
/// See `StorageBase::create_key_btree_index`.
/// Build the configured key index on the base table's `id` column.
pub async fn create_id_key_index(&mut self) -> LanceResult<()> {
self.base.create_configured_key_index().await
}

/// Explicitly create a BTree, independent of the configured merge policy.
pub async fn create_id_btree_index(&mut self) -> LanceResult<()> {
self.base.create_key_btree_index().await
}
Expand Down Expand Up @@ -2851,6 +2859,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3173,6 +3182,7 @@ mod tests {
merge_after_generations: Some(0),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3220,6 +3230,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3272,6 +3283,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3326,6 +3338,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3550,6 +3563,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3615,6 +3629,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3718,6 +3733,7 @@ mod tests {
merge_after_generations: None, // no merge → epoch never reclaimed
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3779,6 +3795,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3821,6 +3838,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -3959,6 +3977,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4048,6 +4067,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4137,6 +4157,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4188,6 +4209,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4275,6 +4297,7 @@ mod tests {
merge_after_generations: Some(3),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4315,6 +4338,7 @@ mod tests {
merge_after_generations: Some(3),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4376,6 +4400,7 @@ mod tests {
merge_after_generations: Some(2),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4536,6 +4561,7 @@ mod tests {
merge_after_generations: None, // count trigger off
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -4678,13 +4704,81 @@ mod tests {
});
}

/// The whole point of the id index: `merge_insert` must take the indexed
/// probe path, not a full-table hash join. Lance's `explain_plan` only
/// renders the full-scan plan and returns `NotSupported` when the job
/// would use a scalar index, so "explain refuses" is the observable
/// signal that the merge will probe. Without the index (and with a
/// ZoneMap, which cannot answer equality exactly) explain succeeds and
/// shows a HashJoin over a LanceScan of the base table.
/// ZoneMap merges use key predicates rather than the full-target merge
/// join. Exercise multiple bounded delete chunks and unusual literal IDs.
#[test]
fn zonemap_merge_preserves_updates_across_delete_chunks_and_retries() {
use crate::KeyIndexType;
use lance::index::DatasetIndexExt;
let dir = TempDir::new().unwrap();
let uri = dir.path().to_string_lossy().to_string();
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut store = RolloutStore::open_with_options(
&uri,
RolloutStoreOptions {
key_index_type: KeyIndexType::Zonemap,
merge_max_generations: Some(1),
..Default::default()
},
)
.await
.unwrap();
let mut ids: Vec<_> = (0..2050).map(|i| format!("key-{i}")).collect();
ids.push("quote' AND false -- \\ unicode-你好".to_string());
for payload in [1u8, 2, 2] {
let records: Vec<_> = ids
.iter()
.map(|id| artifact_record(id, &[payload; 1024]))
.collect();
store.add(&records).await.unwrap();
store.flush().await.unwrap();
assert_eq!(store.cleanup_own_shard().await.unwrap(), 1);
assert_eq!(flushed_generation_count(&store).await, 0);
assert_eq!(
store.base.dataset.count_rows(None).await.unwrap(),
ids.len()
);
let actual = store.list(None, None).await.unwrap();
assert_eq!(actual.len(), ids.len());
let mut scanner = store.base.dataset.scan();
scanner.project(&["binary_payload"]).unwrap();
let mut stream = scanner.try_into_stream().await.unwrap();
while let Some(batch) = stream.try_next().await.unwrap() {
let blobs = batch
.column(0)
.as_any()
.downcast_ref::<arrow_array::LargeBinaryArray>()
.unwrap();
for row in 0..batch.num_rows() {
assert_eq!(blobs.value(row), &[payload; 1024]);
}
}
}
assert!(!store.has_id_btree_index().await.unwrap());
let indices = store.base.dataset.load_indices().await.unwrap();
assert!(indices.iter().any(|i| i.name == ROLLOUT_ID_INDEX_NAME
&& i.index_details
.as_ref()
.is_some_and(|d| d.type_url.ends_with("ZoneMapIndexDetails"))));

// A configured maintenance rebuild must preserve the selected type.
store.create_id_key_index().await.unwrap();
assert!(!store.has_id_btree_index().await.unwrap());
// Changing the policy takes effect on maintenance, not on opening.
drop(store);
let mut store = RolloutStore::open(&uri).await.unwrap();
assert!(!store.has_id_btree_index().await.unwrap());
store
.add(&[artifact_record(&ids[0], &[3; 1024])])
.await
.unwrap();
store.flush().await.unwrap();
store.cleanup_own_shard().await.unwrap();
assert!(store.has_id_btree_index().await.unwrap());
assert_eq!(store.list(None, None).await.unwrap().len(), ids.len());
});
}

#[test]
fn merge_insert_probes_the_id_btree_instead_of_scanning() {
use lance::dataset::MergeInsertBuilder;
Expand Down Expand Up @@ -4825,6 +4919,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: Some(3),
merge_budget: None,
Expand Down Expand Up @@ -4873,6 +4968,7 @@ mod tests {
merge_after_generations: None,
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: Some(2),
merge_budget: None,
Expand Down Expand Up @@ -5102,6 +5198,7 @@ mod tests {
shard_id: Some("budget-contention".to_string()),
merge_max_generations: Some(0),
merge_max_bytes: Some(MIB),
key_index_type: Default::default(),
merge_budget: Some(budget.clone()),
..Default::default()
},
Expand Down Expand Up @@ -5344,6 +5441,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -5445,6 +5543,7 @@ mod tests {
merge_after_generations: None, // disabled
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down Expand Up @@ -5480,6 +5579,7 @@ mod tests {
merge_after_generations: Some(1),
merge_max_generations: None,
merge_max_bytes: None,
key_index_type: Default::default(),
pending_generations_warn: None,
pending_generations_max: None,
merge_budget: None,
Expand Down
Loading
Loading