From 7703c1070911e5caad4a6da08e1d2577eed319e9 Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Fri, 16 Oct 2026 12:33:07 +0530 Subject: [PATCH] build(deps): keep in-process mode off the RPC and HTTP stacks liquid-cache-datafusion-local pulled in arrow-flight, tonic, axum, hyper, h2, reqwest and rustls although in-process mode never uses the network. - Gate liquid_cache_common::rpc behind a new `rpc` feature, off by default; client and server enable it. - Move ExecutionMetricsResponse, a plain serde struct and the only rpc item the in-process crates used, to liquid_cache_common::metrics. - Drop object_store's `http` feature from liquid-cache-datafusion, which does not use it. The client calls parse_url_opts and now enables `http` itself. - Give liquid-cache-common's tests tokio `macros` directly; they had relied on tonic enabling it. Closes #522 --- benchmark/src/lib.rs | 2 +- src/common/Cargo.toml | 12 ++++++++++-- src/common/README.md | 2 ++ src/common/src/lib.rs | 2 ++ src/common/src/metrics.rs | 18 ++++++++++++++++++ src/common/src/rpc.rs | 17 ----------------- src/datafusion-client/Cargo.toml | 4 ++-- src/datafusion-server/Cargo.toml | 2 +- .../src/admin_server/handlers.rs | 2 +- src/datafusion-server/src/service.rs | 2 +- src/datafusion/Cargo.toml | 2 +- src/datafusion/src/utils.rs | 2 +- 12 files changed, 40 insertions(+), 27 deletions(-) create mode 100644 src/common/src/metrics.rs diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index a72185b02..dec471f6d 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -8,7 +8,7 @@ use fastrace::Span; use fastrace::future::FutureExt as _; use liquid_cache::cache::CacheStats; use liquid_cache::cache::{Evict, EvictionPolicy, TranscodeEvict}; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion_server::{ApiResponse, ExecutionStats}; use log::info; use serde::Serialize; diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 582f4bc9c..ba612de07 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -7,14 +7,18 @@ readme = "README.md" description = { workspace = true } repository = { workspace = true } +[features] +# Arrow Flight RPC message types used between the client and the server. +rpc = ["dep:arrow-flight", "dep:prost"] + [dependencies] -arrow-flight = { workspace = true } +arrow-flight = { workspace = true, optional = true } async-trait = { workspace = true } bytes = { workspace = true } chrono = "0.4.45" futures = { workspace = true } object_store = { workspace = true } -prost = { workspace = true } +prost = { workspace = true, optional = true } serde = { workspace = true } thiserror = "2.0.18" tokio = { workspace = true } @@ -22,3 +26,7 @@ url = { workspace = true } [dev-dependencies] tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[package.metadata.docs.rs] +features = ["rpc"] diff --git a/src/common/README.md b/src/common/README.md index 502f42277..9a2875f50 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -3,3 +3,5 @@ Common types and utilities shared across LiquidCache components. This crate contains shared data structures, enums, and utility functions used throughout the LiquidCache ecosystem. It defines the core abstractions for cache modes, schema transformations, and RPC message types that enable communication between client and server components. + +The RPC message types (`rpc` module) are behind the `rpc` feature, which is off by default. The client and server crates enable it; in-process use through `liquid-cache-datafusion-local` does not need it, and so does not depend on Arrow Flight or gRPC. diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index 5b4b43b89..57593084c 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -1,5 +1,7 @@ #![doc = include_str!("../README.md")] +pub mod metrics; pub mod mock_store; +#[cfg(feature = "rpc")] pub mod rpc; pub mod utils; diff --git a/src/common/src/metrics.rs b/src/common/src/metrics.rs new file mode 100644 index 000000000..00960a77e --- /dev/null +++ b/src/common/src/metrics.rs @@ -0,0 +1,18 @@ +//! Execution metrics reported for a query plan. + +#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize, Debug)] +pub struct ExecutionMetricsResponse { + pub pushdown_eval_time: u64, + pub cache_memory_usage: u64, + pub liquid_cache_usage: u64, +} + +impl ExecutionMetricsResponse { + pub fn zero() -> Self { + Self { + pushdown_eval_time: 0, + cache_memory_usage: 0, + liquid_cache_usage: 0, + } + } +} diff --git a/src/common/src/rpc.rs b/src/common/src/rpc.rs index 452d4ef56..0907085ad 100644 --- a/src/common/src/rpc.rs +++ b/src/common/src/rpc.rs @@ -246,20 +246,3 @@ impl ProstMessageExt for FetchResults { } } } - -#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize, Debug)] -pub struct ExecutionMetricsResponse { - pub pushdown_eval_time: u64, - pub cache_memory_usage: u64, - pub liquid_cache_usage: u64, -} - -impl ExecutionMetricsResponse { - pub fn zero() -> Self { - Self { - pushdown_eval_time: 0, - cache_memory_usage: 0, - liquid_cache_usage: 0, - } - } -} diff --git a/src/datafusion-client/Cargo.toml b/src/datafusion-client/Cargo.toml index ddd207cd1..42e041867 100644 --- a/src/datafusion-client/Cargo.toml +++ b/src/datafusion-client/Cargo.toml @@ -17,13 +17,13 @@ arrow-schema = { workspace = true } futures = { workspace = true } tonic = { workspace = true } uuid = { workspace = true } -liquid-cache-common = { workspace = true } +liquid-cache-common = { workspace = true, features = ["rpc"] } liquid-cache-datafusion = { workspace = true } fastrace = { workspace = true } fastrace-tonic = { workspace = true } tower = "0.5.3" tokio = { workspace = true } -object_store = { workspace = true } +object_store = { workspace = true, features = ["http"] } [dev-dependencies] insta = { workspace = true } diff --git a/src/datafusion-server/Cargo.toml b/src/datafusion-server/Cargo.toml index 804b48dc3..a391c48f8 100644 --- a/src/datafusion-server/Cargo.toml +++ b/src/datafusion-server/Cargo.toml @@ -21,7 +21,7 @@ tokio = { workspace = true } url = { workspace = true } liquid-cache-datafusion = { workspace = true } object_store = { workspace = true, features = ["aws", "http"] } -liquid-cache-common = { workspace = true } +liquid-cache-common = { workspace = true, features = ["rpc"] } tempfile = { workspace = true } axum = "0.8.9" serde = { workspace = true } diff --git a/src/datafusion-server/src/admin_server/handlers.rs b/src/datafusion-server/src/admin_server/handlers.rs index 029f928aa..4614fe96e 100644 --- a/src/datafusion-server/src/admin_server/handlers.rs +++ b/src/datafusion-server/src/admin_server/handlers.rs @@ -17,7 +17,7 @@ use datafusion::{ datasource::physical_plan::FileScanConfig, physical_plan::{ExecutionPlan, StatisticsArgs, StatisticsContext}, }; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion::LiquidParquetSource; use log::info; use serde::Serialize; diff --git a/src/datafusion-server/src/service.rs b/src/datafusion-server/src/service.rs index 04c671243..479daca06 100644 --- a/src/datafusion-server/src/service.rs +++ b/src/datafusion-server/src/service.rs @@ -7,7 +7,7 @@ use datafusion::{ }; use liquid_cache::{ByteCache, cache::EvictionPolicy}; use liquid_cache::{cache::HydrationPolicy, cache_policies::CachePolicy}; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion::{ cache::{ColumnLineages, LiquidCacheParquet, LiquidCacheParquetRef}, extract_execution_metrics, diff --git a/src/datafusion/Cargo.toml b/src/datafusion/Cargo.toml index 53e89258f..76f1cbdf4 100644 --- a/src/datafusion/Cargo.toml +++ b/src/datafusion/Cargo.toml @@ -17,7 +17,7 @@ futures = { workspace = true } tokio = { workspace = true } ahash = { workspace = true } log = { workspace = true } -object_store = { workspace = true, features = ["http"] } +object_store = { workspace = true } liquid-cache-common = { workspace = true } liquid-cache = { workspace = true } fastrace = { workspace = true } diff --git a/src/datafusion/src/utils.rs b/src/datafusion/src/utils.rs index e6d903ccf..5d5b3aad6 100644 --- a/src/datafusion/src/utils.rs +++ b/src/datafusion/src/utils.rs @@ -9,7 +9,7 @@ use datafusion::{ datasource::source::DataSourceExec, physical_plan::{ExecutionPlan, metrics::MetricValue}, }; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use parquet::arrow::arrow_reader::RowSelector; use crate::cache::LiquidCacheParquetRef;