diff --git a/crates/lance-context-core/Cargo.toml b/crates/lance-context-core/Cargo.toml index 1c99f876..d621686c 100644 --- a/crates/lance-context-core/Cargo.toml +++ b/crates/lance-context-core/Cargo.toml @@ -26,7 +26,7 @@ serde_json = "1" futures = "0.3" tokio = { version = "1", features = ["sync", "time"] } tracing = "0.1" -uuid = { version = "1.20.0", features = ["v4", "v5"] } +uuid = { version = "1.20.0", features = ["v4", "v5", "v7"] } [dev-dependencies] tempfile = "3" diff --git a/crates/lance-context-core/src/id.rs b/crates/lance-context-core/src/id.rs new file mode 100644 index 00000000..1571025a --- /dev/null +++ b/crates/lance-context-core/src/id.rs @@ -0,0 +1,72 @@ +//! Identifier helpers for context and rollout records. +//! +//! Records are keyed by string ids that callers frequently generate on the +//! write path. Using a **time-ordered** UUID (rather than a fully random +//! [`Uuid::new_v4`]) means the lexical/byte order of ids matches creation +//! order, which keeps appends clustered at the tail of an index instead of +//! scattering them randomly — friendlier to Lance's on-write indexing and to +//! any downstream B-tree/LSM key space. +//! +//! UUIDv7 (RFC 9562) lays out a 48-bit Unix-millisecond timestamp in the high +//! bits followed by random bits, so ids are sortable by time. Ordering across +//! the same millisecond (and across processes/machines) is only approximate: +//! the sub-millisecond bits are random, so two ids minted in the same +//! millisecond have no guaranteed relative order. + +use uuid::Uuid; + +/// Generate a new time-ordered [`UUIDv7`](Uuid::now_v7) as a hyphenated string. +/// +/// The result sorts (lexically) roughly in creation order, making it a good +/// default primary key for append-heavy tables such as the rollout store. +/// +/// ``` +/// let id = lance_context_core::generate_id(); +/// assert_eq!(id.len(), 36); // 32 hex digits + 4 hyphens +/// ``` +#[must_use] +pub fn generate_id() -> String { + new_uuid().to_string() +} + +/// Generate a new time-ordered [`UUIDv7`](Uuid::now_v7). +/// +/// Prefer [`generate_id`] when a `String` id is what you ultimately need; this +/// variant is for callers that want to keep the raw [`Uuid`] (e.g. to compare +/// or to encode differently). +#[must_use] +pub fn new_uuid() -> Uuid { + Uuid::now_v7() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_id_is_a_hyphenated_uuid() { + let id = generate_id(); + // Round-trips as a valid UUID and is the canonical hyphenated form. + let parsed = Uuid::parse_str(&id).expect("valid uuid string"); + assert_eq!(parsed.to_string(), id); + assert_eq!(parsed.get_version_num(), 7); + } + + #[test] + fn ids_are_unique() { + let a = generate_id(); + let b = generate_id(); + assert_ne!(a, b); + } + + #[test] + fn ids_sort_in_creation_order_across_milliseconds() { + // Two ids minted a millisecond apart must sort in creation order, + // because UUIDv7 places the millisecond timestamp in the high bits. + let earlier = new_uuid(); + std::thread::sleep(std::time::Duration::from_millis(2)); + let later = new_uuid(); + assert!(earlier < later, "{earlier} should sort before {later}"); + assert!(earlier.to_string() < later.to_string()); + } +} diff --git a/crates/lance-context-core/src/lib.rs b/crates/lance-context-core/src/lib.rs index 8f4f0826..54cfe0d1 100644 --- a/crates/lance-context-core/src/lib.rs +++ b/crates/lance-context-core/src/lib.rs @@ -5,6 +5,7 @@ mod api_impl; mod context; mod eval; mod export; +mod id; mod namespace; mod record; mod rollout; @@ -23,6 +24,7 @@ pub use export::{ RolloutExample, RolloutResponse, SftExample, SplitConfig, SplitManifest, TokenStats, EXPORT_SCHEMA_VERSION, }; +pub use id::{generate_id, new_uuid}; pub use namespace::{ContextNamespace, PartitionInfo, PartitionSelector, PartitionSpec}; pub use record::{ ContextRecord, LifecycleQueryOptions, MetadataFilter, RecordFilters, RecordPatch, Relationship, diff --git a/python/python/lance_context/__init__.py b/python/python/lance_context/__init__.py index 7854f0d0..bbb5ec56 100644 --- a/python/python/lance_context/__init__.py +++ b/python/python/lance_context/__init__.py @@ -9,6 +9,7 @@ RemoteContext, RolloutStore, __version__, + generate_id, ) from .embeddings import ( # pyright: ignore[reportMissingImports] MultiModalEmbeddingProvider, @@ -24,4 +25,5 @@ "RemoteContext", "RolloutStore", "__version__", + "generate_id", ] diff --git a/python/python/lance_context/api.py b/python/python/lance_context/api.py index 8c9425ed..c9514fde 100644 --- a/python/python/lance_context/api.py +++ b/python/python/lance_context/api.py @@ -21,6 +21,9 @@ from ._internal import ( # pyright: ignore[reportMissingImports] RolloutStore as _RolloutStore, ) +from ._internal import ( # pyright: ignore[reportMissingImports] + generate_id as _generate_id, +) from ._internal import version as _version # pyright: ignore[reportMissingImports] from .embeddings import EmbeddingProvider, _build_provider, supports_media @@ -33,10 +36,21 @@ "RemoteContext", "RolloutStore", "__version__", + "generate_id", ] __version__ = _version() + +def generate_id() -> str: + """Generate a new time-ordered id (UUIDv7) as a string. + + Ids sort roughly in creation order, making them a good default primary key + for append-heavy tables such as the rollout store. + """ + return _generate_id() + + _ARROW_STREAM_MIME = "application/vnd.apache.arrow.stream" _DEFAULT_INGEST_BATCH_SIZE = 1000 _MISSING = object() diff --git a/python/src/lib.rs b/python/src/lib.rs index fa9eaed4..eeb204e1 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -77,6 +77,15 @@ fn version() -> &'static str { env!("CARGO_PKG_VERSION") } +/// Generate a new time-ordered id (UUIDv7) as a string. +/// +/// Ids sort roughly in creation order, making them a good default primary key +/// for append-heavy tables such as the rollout store. +#[pyfunction] +fn generate_id() -> String { + lance_context_core::generate_id() +} + #[pyclass] struct Context { inner: RustContext, @@ -2517,6 +2526,7 @@ fn rollout_record_to_json(record: &RolloutRecordDto) -> PyResult { #[pymodule] fn _internal(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(version, m)?)?; + m.add_function(wrap_pyfunction!(generate_id, m)?)?; m.add_class::()?; m.add_class::()?; m.add_class::()?;