diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index a72185b0..dec471f6 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -8,7 +8,7 @@ use fastrace::Span; use fastrace::future::FutureExt as _; use liquid_cache::cache::CacheStats; use liquid_cache::cache::{Evict, EvictionPolicy, TranscodeEvict}; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion_server::{ApiResponse, ExecutionStats}; use log::info; use serde::Serialize; diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 582f4bc9..ba612de0 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -7,14 +7,18 @@ readme = "README.md" description = { workspace = true } repository = { workspace = true } +[features] +# Arrow Flight RPC message types used between the client and the server. +rpc = ["dep:arrow-flight", "dep:prost"] + [dependencies] -arrow-flight = { workspace = true } +arrow-flight = { workspace = true, optional = true } async-trait = { workspace = true } bytes = { workspace = true } chrono = "0.4.45" futures = { workspace = true } object_store = { workspace = true } -prost = { workspace = true } +prost = { workspace = true, optional = true } serde = { workspace = true } thiserror = "2.0.18" tokio = { workspace = true } @@ -22,3 +26,7 @@ url = { workspace = true } [dev-dependencies] tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[package.metadata.docs.rs] +features = ["rpc"] diff --git a/src/common/README.md b/src/common/README.md index 502f4227..9a2875f5 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -3,3 +3,5 @@ Common types and utilities shared across LiquidCache components. This crate contains shared data structures, enums, and utility functions used throughout the LiquidCache ecosystem. It defines the core abstractions for cache modes, schema transformations, and RPC message types that enable communication between client and server components. + +The RPC message types (`rpc` module) are behind the `rpc` feature, which is off by default. The client and server crates enable it; in-process use through `liquid-cache-datafusion-local` does not need it, and so does not depend on Arrow Flight or gRPC. diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index 5b4b43b8..57593084 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -1,5 +1,7 @@ #![doc = include_str!("../README.md")] +pub mod metrics; pub mod mock_store; +#[cfg(feature = "rpc")] pub mod rpc; pub mod utils; diff --git a/src/common/src/metrics.rs b/src/common/src/metrics.rs new file mode 100644 index 00000000..00960a77 --- /dev/null +++ b/src/common/src/metrics.rs @@ -0,0 +1,18 @@ +//! Execution metrics reported for a query plan. + +#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize, Debug)] +pub struct ExecutionMetricsResponse { + pub pushdown_eval_time: u64, + pub cache_memory_usage: u64, + pub liquid_cache_usage: u64, +} + +impl ExecutionMetricsResponse { + pub fn zero() -> Self { + Self { + pushdown_eval_time: 0, + cache_memory_usage: 0, + liquid_cache_usage: 0, + } + } +} diff --git a/src/common/src/rpc.rs b/src/common/src/rpc.rs index 452d4ef5..0907085a 100644 --- a/src/common/src/rpc.rs +++ b/src/common/src/rpc.rs @@ -246,20 +246,3 @@ impl ProstMessageExt for FetchResults { } } } - -#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize, Debug)] -pub struct ExecutionMetricsResponse { - pub pushdown_eval_time: u64, - pub cache_memory_usage: u64, - pub liquid_cache_usage: u64, -} - -impl ExecutionMetricsResponse { - pub fn zero() -> Self { - Self { - pushdown_eval_time: 0, - cache_memory_usage: 0, - liquid_cache_usage: 0, - } - } -} diff --git a/src/datafusion-client/Cargo.toml b/src/datafusion-client/Cargo.toml index ddd207cd..42e04186 100644 --- a/src/datafusion-client/Cargo.toml +++ b/src/datafusion-client/Cargo.toml @@ -17,13 +17,13 @@ arrow-schema = { workspace = true } futures = { workspace = true } tonic = { workspace = true } uuid = { workspace = true } -liquid-cache-common = { workspace = true } +liquid-cache-common = { workspace = true, features = ["rpc"] } liquid-cache-datafusion = { workspace = true } fastrace = { workspace = true } fastrace-tonic = { workspace = true } tower = "0.5.3" tokio = { workspace = true } -object_store = { workspace = true } +object_store = { workspace = true, features = ["http"] } [dev-dependencies] insta = { workspace = true } diff --git a/src/datafusion-server/Cargo.toml b/src/datafusion-server/Cargo.toml index 804b48dc..a391c48f 100644 --- a/src/datafusion-server/Cargo.toml +++ b/src/datafusion-server/Cargo.toml @@ -21,7 +21,7 @@ tokio = { workspace = true } url = { workspace = true } liquid-cache-datafusion = { workspace = true } object_store = { workspace = true, features = ["aws", "http"] } -liquid-cache-common = { workspace = true } +liquid-cache-common = { workspace = true, features = ["rpc"] } tempfile = { workspace = true } axum = "0.8.9" serde = { workspace = true } diff --git a/src/datafusion-server/src/admin_server/handlers.rs b/src/datafusion-server/src/admin_server/handlers.rs index 029f928a..4614fe96 100644 --- a/src/datafusion-server/src/admin_server/handlers.rs +++ b/src/datafusion-server/src/admin_server/handlers.rs @@ -17,7 +17,7 @@ use datafusion::{ datasource::physical_plan::FileScanConfig, physical_plan::{ExecutionPlan, StatisticsArgs, StatisticsContext}, }; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion::LiquidParquetSource; use log::info; use serde::Serialize; diff --git a/src/datafusion-server/src/service.rs b/src/datafusion-server/src/service.rs index 04c67124..479daca0 100644 --- a/src/datafusion-server/src/service.rs +++ b/src/datafusion-server/src/service.rs @@ -7,7 +7,7 @@ use datafusion::{ }; use liquid_cache::{ByteCache, cache::EvictionPolicy}; use liquid_cache::{cache::HydrationPolicy, cache_policies::CachePolicy}; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use liquid_cache_datafusion::{ cache::{ColumnLineages, LiquidCacheParquet, LiquidCacheParquetRef}, extract_execution_metrics, diff --git a/src/datafusion/Cargo.toml b/src/datafusion/Cargo.toml index 53e89258..76f1cbdf 100644 --- a/src/datafusion/Cargo.toml +++ b/src/datafusion/Cargo.toml @@ -17,7 +17,7 @@ futures = { workspace = true } tokio = { workspace = true } ahash = { workspace = true } log = { workspace = true } -object_store = { workspace = true, features = ["http"] } +object_store = { workspace = true } liquid-cache-common = { workspace = true } liquid-cache = { workspace = true } fastrace = { workspace = true } diff --git a/src/datafusion/src/utils.rs b/src/datafusion/src/utils.rs index e6d903cc..5d5b3aad 100644 --- a/src/datafusion/src/utils.rs +++ b/src/datafusion/src/utils.rs @@ -9,7 +9,7 @@ use datafusion::{ datasource::source::DataSourceExec, physical_plan::{ExecutionPlan, metrics::MetricValue}, }; -use liquid_cache_common::rpc::ExecutionMetricsResponse; +use liquid_cache_common::metrics::ExecutionMetricsResponse; use parquet::arrow::arrow_reader::RowSelector; use crate::cache::LiquidCacheParquetRef;