From b50a754449137217257c522e79e72d3f1ebeac16 Mon Sep 17 00:00:00 2001 From: Beinan Date: Tue, 14 Jul 2026 05:04:16 +0000 Subject: [PATCH 1/2] feat(core): add UUIDv7 id-generation util + Python binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records are keyed by caller-supplied string ids. Provide a shared time-ordered id generator so callers default to sortable keys instead of fully-random v4: `generate_id()` / `new_uuid()` in lance-context-core, re-exported at the crate root, plus a `generate_id()` pyfunction exposed as `lance_context.generate_id` in the Python package. UUIDv7 puts a 48-bit Unix-ms timestamp in the high bits, so ids sort roughly in creation order — keeping appends clustered at an index tail rather than scattering them (friendlier to Lance's on-write indexing). Co-Authored-By: Claude Opus 4 --- crates/lance-context-core/Cargo.toml | 2 +- crates/lance-context-core/src/id.rs | 72 +++++++++++++++++++++++++ crates/lance-context-core/src/lib.rs | 2 + python/python/lance_context/__init__.py | 2 + python/python/lance_context/api.py | 13 +++++ python/src/lib.rs | 10 ++++ 6 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 crates/lance-context-core/src/id.rs diff --git a/crates/lance-context-core/Cargo.toml b/crates/lance-context-core/Cargo.toml index 1c99f876..d621686c 100644 --- a/crates/lance-context-core/Cargo.toml +++ b/crates/lance-context-core/Cargo.toml @@ -26,7 +26,7 @@ serde_json = "1" futures = "0.3" tokio = { version = "1", features = ["sync", "time"] } tracing = "0.1" -uuid = { version = "1.20.0", features = ["v4", "v5"] } +uuid = { version = "1.20.0", features = ["v4", "v5", "v7"] } [dev-dependencies] tempfile = "3" diff --git a/crates/lance-context-core/src/id.rs b/crates/lance-context-core/src/id.rs new file mode 100644 index 00000000..1571025a --- /dev/null +++ b/crates/lance-context-core/src/id.rs @@ -0,0 +1,72 @@ +//! Identifier helpers for context and rollout records. +//! +//! Records are keyed by string ids that callers frequently generate on the +//! write path. Using a **time-ordered** UUID (rather than a fully random +//! [`Uuid::new_v4`]) means the lexical/byte order of ids matches creation +//! order, which keeps appends clustered at the tail of an index instead of +//! scattering them randomly — friendlier to Lance's on-write indexing and to +//! any downstream B-tree/LSM key space. +//! +//! UUIDv7 (RFC 9562) lays out a 48-bit Unix-millisecond timestamp in the high +//! bits followed by random bits, so ids are sortable by time. Ordering across +//! the same millisecond (and across processes/machines) is only approximate: +//! the sub-millisecond bits are random, so two ids minted in the same +//! millisecond have no guaranteed relative order. + +use uuid::Uuid; + +/// Generate a new time-ordered [`UUIDv7`](Uuid::now_v7) as a hyphenated string. +/// +/// The result sorts (lexically) roughly in creation order, making it a good +/// default primary key for append-heavy tables such as the rollout store. +/// +/// ``` +/// let id = lance_context_core::generate_id(); +/// assert_eq!(id.len(), 36); // 32 hex digits + 4 hyphens +/// ``` +#[must_use] +pub fn generate_id() -> String { + new_uuid().to_string() +} + +/// Generate a new time-ordered [`UUIDv7`](Uuid::now_v7). +/// +/// Prefer [`generate_id`] when a `String` id is what you ultimately need; this +/// variant is for callers that want to keep the raw [`Uuid`] (e.g. to compare +/// or to encode differently). +#[must_use] +pub fn new_uuid() -> Uuid { + Uuid::now_v7() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_id_is_a_hyphenated_uuid() { + let id = generate_id(); + // Round-trips as a valid UUID and is the canonical hyphenated form. + let parsed = Uuid::parse_str(&id).expect("valid uuid string"); + assert_eq!(parsed.to_string(), id); + assert_eq!(parsed.get_version_num(), 7); + } + + #[test] + fn ids_are_unique() { + let a = generate_id(); + let b = generate_id(); + assert_ne!(a, b); + } + + #[test] + fn ids_sort_in_creation_order_across_milliseconds() { + // Two ids minted a millisecond apart must sort in creation order, + // because UUIDv7 places the millisecond timestamp in the high bits. + let earlier = new_uuid(); + std::thread::sleep(std::time::Duration::from_millis(2)); + let later = new_uuid(); + assert!(earlier < later, "{earlier} should sort before {later}"); + assert!(earlier.to_string() < later.to_string()); + } +} diff --git a/crates/lance-context-core/src/lib.rs b/crates/lance-context-core/src/lib.rs index 8f4f0826..54cfe0d1 100644 --- a/crates/lance-context-core/src/lib.rs +++ b/crates/lance-context-core/src/lib.rs @@ -5,6 +5,7 @@ mod api_impl; mod context; mod eval; mod export; +mod id; mod namespace; mod record; mod rollout; @@ -23,6 +24,7 @@ pub use export::{ RolloutExample, RolloutResponse, SftExample, SplitConfig, SplitManifest, TokenStats, EXPORT_SCHEMA_VERSION, }; +pub use id::{generate_id, new_uuid}; pub use namespace::{ContextNamespace, PartitionInfo, PartitionSelector, PartitionSpec}; pub use record::{ ContextRecord, LifecycleQueryOptions, MetadataFilter, RecordFilters, RecordPatch, Relationship, diff --git a/python/python/lance_context/__init__.py b/python/python/lance_context/__init__.py index 7854f0d0..bbb5ec56 100644 --- a/python/python/lance_context/__init__.py +++ b/python/python/lance_context/__init__.py @@ -9,6 +9,7 @@ RemoteContext, RolloutStore, __version__, + generate_id, ) from .embeddings import ( # pyright: ignore[reportMissingImports] MultiModalEmbeddingProvider, @@ -24,4 +25,5 @@ "RemoteContext", "RolloutStore", "__version__", + "generate_id", ] diff --git a/python/python/lance_context/api.py b/python/python/lance_context/api.py index 8c9425ed..a56bedb6 100644 --- a/python/python/lance_context/api.py +++ b/python/python/lance_context/api.py @@ -22,6 +22,9 @@ RolloutStore as _RolloutStore, ) from ._internal import version as _version # pyright: ignore[reportMissingImports] +from ._internal import ( # pyright: ignore[reportMissingImports] + generate_id as _generate_id, +) from .embeddings import EmbeddingProvider, _build_provider, supports_media __all__ = [ @@ -33,10 +36,20 @@ "RemoteContext", "RolloutStore", "__version__", + "generate_id", ] __version__ = _version() + +def generate_id() -> str: + """Generate a new time-ordered id (UUIDv7) as a string. + + Ids sort roughly in creation order, making them a good default primary key + for append-heavy tables such as the rollout store. + """ + return _generate_id() + _ARROW_STREAM_MIME = "application/vnd.apache.arrow.stream" _DEFAULT_INGEST_BATCH_SIZE = 1000 _MISSING = object() diff --git a/python/src/lib.rs b/python/src/lib.rs index fa9eaed4..eeb204e1 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -77,6 +77,15 @@ fn version() -> &'static str { env!("CARGO_PKG_VERSION") } +/// Generate a new time-ordered id (UUIDv7) as a string. +/// +/// Ids sort roughly in creation order, making them a good default primary key +/// for append-heavy tables such as the rollout store. +#[pyfunction] +fn generate_id() -> String { + lance_context_core::generate_id() +} + #[pyclass] struct Context { inner: RustContext, @@ -2517,6 +2526,7 @@ fn rollout_record_to_json(record: &RolloutRecordDto) -> PyResult { #[pymodule] fn _internal(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(version, m)?)?; + m.add_function(wrap_pyfunction!(generate_id, m)?)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From eda183ffb6c484c441bd90ac1b38c97aa2761f93 Mon Sep 17 00:00:00 2001 From: Beinan Date: Tue, 14 Jul 2026 05:28:26 +0000 Subject: [PATCH 2/2] style(python): satisfy ruff format + import order in api.py Two blank lines after generate_id() and sort the _internal imports so `generate_id` precedes `version`, per ruff. Co-Authored-By: Claude Opus 4 --- python/python/lance_context/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/python/lance_context/api.py b/python/python/lance_context/api.py index a56bedb6..c9514fde 100644 --- a/python/python/lance_context/api.py +++ b/python/python/lance_context/api.py @@ -21,10 +21,10 @@ from ._internal import ( # pyright: ignore[reportMissingImports] RolloutStore as _RolloutStore, ) -from ._internal import version as _version # pyright: ignore[reportMissingImports] from ._internal import ( # pyright: ignore[reportMissingImports] generate_id as _generate_id, ) +from ._internal import version as _version # pyright: ignore[reportMissingImports] from .embeddings import EmbeddingProvider, _build_provider, supports_media __all__ = [ @@ -50,6 +50,7 @@ def generate_id() -> str: """ return _generate_id() + _ARROW_STREAM_MIME = "application/vnd.apache.arrow.stream" _DEFAULT_INGEST_BATCH_SIZE = 1000 _MISSING = object()