chore: upgrade Lance to 13 and Arrow to 58

crates/lance-graph-benches/Cargo.toml

@@ -7,9 +7,9 @@ description = "Benchmarks for lance-graph (not published)"
 
 [dependencies]
 lance-graph = { path = "../lance-graph", version = "0.5.3" }
-lance = "1.0.0"
-arrow-array = "56.2"
-arrow-schema = "56.2"
+lance = "13.0.0"
+arrow-array = "58.4"
+arrow-schema = "58.4"
 criterion = { version = "0.5", features = ["async", "async_tokio", "html_reports"] }
 futures = "0.3"
 tempfile = "3"

crates/lance-graph-catalog/Cargo.toml

@@ -11,10 +11,10 @@ keywords = ["lance", "graph", "catalog", "namespace"]
 categories = ["database", "data-structures", "science"]
 
 [dependencies]
-arrow-schema = "56.2"
+arrow-schema = "58.4"
 async-trait = "0.1"
-datafusion = { version = "50.3", default-features = false }
-lance-namespace = "1.0.1"
+datafusion = { version = "54.1", default-features = false }
+lance-namespace = "13.0.0"
 reqwest = { version = "0.12", features = ["json"], optional = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/lance-graph-catalog/src/namespace/directory.rs

@@ -3,7 +3,6 @@
 use async_trait::async_trait;
 use lance_namespace::models::{DescribeTableRequest, DescribeTableResponse};
 use lance_namespace::{Error as NamespaceError, LanceNamespace, Result};
-use snafu::location;
 
 /// A namespace that resolves table names relative to a base directory or URI.
 #[derive(Debug, Clone)]
@@ -39,18 +38,14 @@ impl LanceNamespace for DirNamespace {
         let id = request.id.ok_or_else(|| {
             NamespaceError::invalid_input(
                 "DirNamespace requires the table identifier to be provided",
-                location!(),
             )
         })?;
 
         if id.len() != 1 {
-            return Err(NamespaceError::invalid_input(
-                format!(
-                    "DirNamespace expects identifiers with a single component, got {:?}",
-                    id
-                ),
-                location!(),
-            ));
+            return Err(NamespaceError::invalid_input(format!(
+                "DirNamespace expects identifiers with a single component, got {:?}",
+                id
+            )));
         }
 
         let table_name = &id[0];

crates/lance-graph-catalog/src/source_catalog.rs

@@ -3,7 +3,6 @@
 
 //! Context-free source catalog for DataFusion logical planning.
 
-use std::any::Any;
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -92,9 +91,6 @@ impl SimpleTableSource {
 }
 
 impl TableSource for SimpleTableSource {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }

crates/lance-graph-python/Cargo.toml

@@ -10,14 +10,14 @@ name = "_internal"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "56.2", features = ["pyarrow"] }
-arrow-array = "56.2"
-arrow-schema = "56.2"
-arrow-ipc = "56.2"
-datafusion = { version = "50.3", default-features = false }
+arrow = { version = "58.4", features = ["pyarrow"] }
+arrow-array = "58.4"
+arrow-schema = "58.4"
+arrow-ipc = "58.4"
+datafusion = { version = "54.1", default-features = false }
 futures = "0.3"
 lance-graph = { path = "../lance-graph" }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39", "py-clone"] }
+pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "py-clone"] }
 tokio = { version = "1.37", features = ["rt-multi-thread", "macros"] }

crates/lance-graph-python/src/catalog.rs

@@ -20,7 +20,7 @@ use crate::RT;
 
 // ---- Python wrapper for CatalogInfo ----
 
-#[pyclass(name = "CatalogInfo", module = "lance.graph")]
+#[pyclass(name = "CatalogInfo", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct PyCatalogInfo {
     inner: CatalogInfo,
@@ -45,7 +45,7 @@ impl PyCatalogInfo {
 
 // ---- Python wrapper for SchemaInfo ----
 
-#[pyclass(name = "SchemaInfo", module = "lance.graph")]
+#[pyclass(name = "SchemaInfo", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct PySchemaInfo {
     inner: SchemaInfo,
@@ -78,7 +78,7 @@ impl PySchemaInfo {
 
 // ---- Python wrapper for TableInfo ----
 
-#[pyclass(name = "TableInfo", module = "lance.graph")]
+#[pyclass(name = "TableInfo", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct PyTableInfo {
     inner: TableInfo,
@@ -250,9 +250,7 @@ impl PyUnityCatalog {
         let connector = self.connector.clone();
         let cat = catalog_name.to_string();
         let result = RT
-            .block_on(Some(py), async move {
-                connector.list_schemas(&cat).await
-            })?
+            .block_on(Some(py), async move { connector.list_schemas(&cat).await })?
             .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
         Ok(result
             .into_iter()
@@ -278,9 +276,10 @@ impl PyUnityCatalog {
         let cat = catalog_name.to_string();
         let sch = schema_name.to_string();
         let result = RT
-            .block_on(Some(py), async move {
-                connector.list_tables(&cat, &sch).await
-            })?
+            .block_on(
+                Some(py),
+                async move { connector.list_tables(&cat, &sch).await },
+            )?
             .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
         Ok(result
             .into_iter()

crates/lance-graph-python/src/executor.rs

@@ -58,10 +58,10 @@ impl BackgroundExecutor {
         T::Output: Send + 'static,
     {
         if let Some(py) = py {
-            py.allow_threads(|| self.spawn_impl(task))
+            py.detach(|| self.spawn_impl(task))
         } else {
-            // Python::with_gil is a no-op if the GIL is already held by the thread.
-            Python::with_gil(|py| py.allow_threads(|| self.spawn_impl(task)))
+            // Python::attach is a no-op if the GIL is already held by the thread.
+            Python::attach(|py| py.detach(|| self.spawn_impl(task)))
         }
     }
 
@@ -86,7 +86,7 @@ impl BackgroundExecutor {
 
         loop {
             // Check for keyboard interrupts
-            match Python::with_gil(|py| py.check_signals()) {
+            match Python::attach(|py| py.check_signals()) {
                 Ok(_) => {}
                 Err(err) => {
                     handle.abort();
@@ -113,13 +113,13 @@ impl BackgroundExecutor {
         T::Output: Send + 'static,
     {
         if let Some(py) = py {
-            py.allow_threads(|| {
+            py.detach(|| {
                 self.runtime.spawn(task);
             })
         } else {
-            // Python::with_gil is a no-op if the GIL is already held by the thread.
-            Python::with_gil(|py| {
-                py.allow_threads(|| {
+            // Python::attach is a no-op if the GIL is already held by the thread.
+            Python::attach(|py| {
+                py.detach(|| {
                     self.runtime.spawn(task);
                 })
             })
@@ -143,10 +143,10 @@ impl BackgroundExecutor {
     {
         let future = Self::result_or_interrupt(future);
         if let Some(py) = py {
-            py.allow_threads(move || self.runtime.block_on(future))
+            py.detach(move || self.runtime.block_on(future))
         } else {
-            // Python::with_gil is a no-op if the GIL is already held by the thread.
-            Python::with_gil(|py| py.allow_threads(|| self.runtime.block_on(future)))
+            // Python::attach is a no-op if the GIL is already held by the thread.
+            Python::attach(|py| py.detach(|| self.runtime.block_on(future)))
         }
     }
 
@@ -158,7 +158,7 @@ impl BackgroundExecutor {
         let interrupt_future = async {
             loop {
                 // Check for keyboard interrupts
-                match Python::with_gil(|py| py.check_signals()) {
+                match Python::attach(|py| py.check_signals()) {
                     Ok(_) => {
                         // Wait for 100ms before checking signals again
                         tokio::time::sleep(SIGNAL_CHECK_INTERVAL).await;

crates/lance-graph-python/src/graph.rs

@@ -27,8 +27,7 @@ use lance_graph::{
     ast::{DistanceMetric as RustDistanceMetric, GraphPattern, ReadingClause},
     CypherQuery as RustCypherQuery, ExecutionStrategy as RustExecutionStrategy,
     GraphConfig as RustGraphConfig, GraphError as RustGraphError, InMemoryCatalog,
-    SqlDialect as RustSqlDialect, SqlQuery as RustSqlQuery,
-    VectorSearch as RustVectorSearch,
+    SqlDialect as RustSqlDialect, SqlQuery as RustSqlQuery, VectorSearch as RustVectorSearch,
 };
 use pyo3::{
     exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
@@ -42,7 +41,7 @@ use crate::namespace::PyDirNamespace;
 use crate::RT;
 
 /// Execution strategy for Cypher queries
-#[pyclass(name = "ExecutionStrategy", module = "lance.graph")]
+#[pyclass(name = "ExecutionStrategy", module = "lance.graph", from_py_object)]
 #[derive(Clone, Copy)]
 pub enum ExecutionStrategy {
     /// Use DataFusion query planner (default, full feature support)
@@ -61,7 +60,7 @@ impl From<ExecutionStrategy> for RustExecutionStrategy {
 }
 
 /// SQL dialect for generating SQL from Cypher queries
-#[pyclass(name = "SqlDialect", module = "lance.graph")]
+#[pyclass(name = "SqlDialect", module = "lance.graph", from_py_object)]
 #[derive(Clone, Copy)]
 pub enum SqlDialect {
     /// Generic SQL (DataFusion default dialect)
@@ -89,7 +88,7 @@ impl From<SqlDialect> for RustSqlDialect {
 }
 
 /// Distance metric for vector similarity search
-#[pyclass(name = "DistanceMetric", module = "lance.graph")]
+#[pyclass(name = "DistanceMetric", module = "lance.graph", from_py_object)]
 #[derive(Clone, Copy)]
 pub enum DistanceMetric {
     /// L2 (Euclidean) distance - smaller is more similar
@@ -129,7 +128,7 @@ impl From<DistanceMetric> for RustDistanceMetric {
 /// ...     .metric(DistanceMetric.Cosine) \
 /// ...     .top_k(10) \
 /// ...     .search(table)
-#[pyclass(name = "VectorSearch", module = "lance.graph")]
+#[pyclass(name = "VectorSearch", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct VectorSearch {
     inner: RustVectorSearch,
@@ -290,7 +289,7 @@ impl VectorSearch {
     ///     If query vector is not set or column not found
     /// RuntimeError
     ///     If search execution fails
-    fn search(&self, py: Python, table: &Bound<'_, PyAny>) -> PyResult<PyObject> {
+    fn search(&self, py: Python, table: &Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
         let batch = python_any_to_record_batch(table)?;
         let batch = normalize_record_batch(batch)?;
 
@@ -322,7 +321,7 @@ pub(crate) fn graph_error_to_pyerr(err: RustGraphError) -> PyErr {
 }
 
 /// Graph configuration for interpreting Lance datasets as property graphs
-#[pyclass(name = "GraphConfig", module = "lance.graph")]
+#[pyclass(name = "GraphConfig", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct GraphConfig {
     inner: RustGraphConfig,
@@ -358,7 +357,7 @@ impl GraphConfig {
 }
 
 /// Builder for GraphConfig
-#[pyclass(name = "GraphConfigBuilder", module = "lance.graph")]
+#[pyclass(name = "GraphConfigBuilder", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct GraphConfigBuilder {
     inner: lance_graph::config::GraphConfigBuilder,
@@ -434,7 +433,7 @@ impl GraphConfigBuilder {
 }
 
 /// Cypher query interface for Lance datasets
-#[pyclass(name = "CypherQuery", module = "lance.graph")]
+#[pyclass(name = "CypherQuery", module = "lance.graph", from_py_object)]
 #[derive(Clone)]
 pub struct CypherQuery {
     inner: RustCypherQuery,
@@ -591,7 +590,7 @@ impl CypherQuery {
         py: Python,
         datasets: &Bound<'_, PyDict>,
         strategy: Option<ExecutionStrategy>,
-    ) -> PyResult<PyObject> {
+    ) -> PyResult<Py<PyAny>> {
         // Convert datasets to Arrow batches while holding the GIL
         let arrow_datasets = python_datasets_to_batches(datasets)?;
 
@@ -633,7 +632,7 @@ impl CypherQuery {
         py: Python,
         namespace: &Bound<'_, PyDirNamespace>,
         strategy: Option<ExecutionStrategy>,
-    ) -> PyResult<PyObject> {
+    ) -> PyResult<Py<PyAny>> {
         let rust_strategy = strategy.map(|s| s.into());
         let inner_query = self.inner.clone();
         let namespace_arc = namespace.borrow().inner.clone();
@@ -744,7 +743,7 @@ impl CypherQuery {
         py: Python,
         datasets: &Bound<'_, PyDict>,
         vector_search: &VectorSearch,
-    ) -> PyResult<PyObject> {
+    ) -> PyResult<Py<PyAny>> {
         if vector_search.use_lance_index {
             if let Some(result) =
                 try_execute_with_lance_index(py, &self.inner, datasets, vector_search)?
@@ -796,7 +795,7 @@ fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult<JsonValue> {
     }
 }
 
-fn json_to_python(py: Python, value: &JsonValue) -> PyResult<PyObject> {
+fn json_to_python(py: Python, value: &JsonValue) -> PyResult<Py<PyAny>> {
     match value {
         JsonValue::Null => Ok(py.None()),
         JsonValue::Bool(b) => {
@@ -1106,7 +1105,7 @@ fn python_any_to_record_batch(value: &Bound<'_, PyAny>) -> PyResult<RecordBatch>
 fn record_batch_to_python_table(
     py: Python,
     batch: &arrow_array::RecordBatch,
-) -> PyResult<PyObject> {
+) -> PyResult<Py<PyAny>> {
     use arrow::pyarrow::ToPyArrow;
     use pyo3::types::PyList;
 
@@ -1273,7 +1272,7 @@ impl CypherEngine {
     /// --------
     /// >>> result = engine.execute("MATCH (p:Person) WHERE p.age > 30 RETURN p.name")
     /// >>> print(result.to_pandas())
-    fn execute(&self, py: Python, query: &str) -> PyResult<PyObject> {
+    fn execute(&self, py: Python, query: &str) -> PyResult<Py<PyAny>> {
         // Parse the query
         let cypher_query = RustCypherQuery::new(query)
             .map_err(graph_error_to_pyerr)?
@@ -1325,7 +1324,7 @@ impl CypherEngine {
         py: Python,
         query: &str,
         vector_search: &VectorSearch,
-    ) -> PyResult<PyObject> {
+    ) -> PyResult<Py<PyAny>> {
         // Parse query and execute with cached catalog/context
         let cypher_query = RustCypherQuery::new(query)
             .map_err(graph_error_to_pyerr)?
@@ -1418,7 +1417,7 @@ impl SqlQuery {
     /// ------
     /// RuntimeError
     ///     If query execution fails
-    fn execute(&self, py: Python, datasets: &Bound<'_, PyDict>) -> PyResult<PyObject> {
+    fn execute(&self, py: Python, datasets: &Bound<'_, PyDict>) -> PyResult<Py<PyAny>> {
         let arrow_datasets = python_datasets_to_batches(datasets)?;
         let inner = self.inner.clone();
 
@@ -1553,7 +1552,7 @@ impl SqlEngine {
     /// ------
     /// RuntimeError
     ///     If query execution fails
-    fn execute(&self, py: Python, sql: &str) -> PyResult<PyObject> {
+    fn execute(&self, py: Python, sql: &str) -> PyResult<Py<PyAny>> {
         let query = RustSqlQuery::new(sql);
         let context = self.context.as_ref().clone();