Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ parquet = { version = "59.2.0", features = [
"variant_experimental",
] }
parquet-variant-json = { version = "59.2.0" }
parquet-variant-compute = { version = "59.2.0" }
datafusion = { version = "55.0.0" }
datafusion-datasource = { version = "55.0.0" }
datafusion-common = { version = "55.0.0" }
Expand Down
4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,7 @@ For performance testing, disable background transcoding:

```rust
let (ctx, _) = LiquidCacheLocalBuilder::new()
.with_squeeze_policy(Box::new(
squeeze_policies::Evict,
))
.with_eviction_policy(Box::new(Evict))
.build(config)
.await?;
```
Expand Down
12 changes: 12 additions & 0 deletions benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,18 @@ cargo run --release --bin in_process -- \
## In process mode
The benchmark uses a JSON manifest file to describe the data tables and queries to run.

### Cache representation ablation

The core cache benchmark can compare direct Arrow eviction with the Liquid
transcode-and-evict path. Use the same input and memory budget for both runs:

```bash
cargo bench -p liquid-cache --bench cache_storage -- \
--parquet ../../examples/nano_hits.parquet --max-memory-mb 1 --mode arrow
cargo bench -p liquid-cache --bench cache_storage -- \
--parquet ../../examples/nano_hits.parquet --max-memory-mb 1 --mode liquid
```

### JSON Format

```json
Expand Down
4 changes: 2 additions & 2 deletions benchmark/bench_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::process::exit(1);
}));
}
let squeeze_policy = args.cache_mode.to_squeeze_policy();
let eviction_policy = args.cache_mode.to_eviction_policy();

// LiquidCache server mode
let ctx = LiquidCacheService::context()?;
Expand All @@ -76,7 +76,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
max_memory_bytes,
args.disk_cache_dir.clone(),
Box::new(LiquidPolicy::new()),
squeeze_policy,
eviction_policy,
Box::new(NoHydration::new()),
)
.await?;
Expand Down
21 changes: 4 additions & 17 deletions benchmark/src/inprocess_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use datafusion::parquet::{
};
use datafusion::prelude::{SessionConfig, SessionContext};
use liquid_cache::cache::NoHydration;
use liquid_cache::cache::squeeze_policies::{Evict, TranscodeEvict, TranscodeSqueezeEvict};
use liquid_cache::cache::{Evict, TranscodeEvict};
use liquid_cache::cache_policies::LiquidPolicy;
use liquid_cache_datafusion::{LiquidCacheParquetRef, extract_execution_metrics};
use liquid_cache_datafusion_local::LiquidCacheLocalBuilder;
Expand Down Expand Up @@ -190,7 +190,6 @@ pub enum InProcessBenchmarkMode {
Arrow,
#[default]
Liquid,
LiquidNoSqueeze,
}

impl std::str::FromStr for InProcessBenchmarkMode {
Expand All @@ -202,10 +201,9 @@ impl std::str::FromStr for InProcessBenchmarkMode {
"datafusion-default" | "datafusion" => InProcessBenchmarkMode::DataFusionDefault,
"arrow" => InProcessBenchmarkMode::Arrow,
"liquid" => InProcessBenchmarkMode::Liquid,
"liquid-no-squeeze" => InProcessBenchmarkMode::LiquidNoSqueeze,
_ => {
return Err(format!(
"Invalid in-process benchmark mode: {s}, must be one of: parquet, datafusion-default, arrow, liquid, liquid-no-squeeze"
"Invalid in-process benchmark mode: {s}, must be one of: parquet, datafusion-default, arrow, liquid"
));
}
})
Expand Down Expand Up @@ -350,7 +348,7 @@ impl InProcessBenchmarkRunner {
.with_cache_dir(cache_dir)
.with_cache_policy(Box::new(LiquidPolicy::new()))
.with_hydration_policy(Box::new(NoHydration::new()))
.with_squeeze_policy(Box::new(Evict))
.with_eviction_policy(Box::new(Evict))
.build(session_config)
.await?;
(v.0, Some(v.1))
Expand All @@ -361,18 +359,7 @@ impl InProcessBenchmarkRunner {
.with_cache_dir(cache_dir)
.with_cache_policy(Box::new(LiquidPolicy::new()))
.with_hydration_policy(Box::new(NoHydration::new()))
.with_squeeze_policy(Box::new(TranscodeSqueezeEvict))
.build(session_config)
.await?;
(v.0, Some(v.1))
}
InProcessBenchmarkMode::LiquidNoSqueeze => {
let v = LiquidCacheLocalBuilder::new()
.with_max_memory_bytes(cache_size)
.with_cache_dir(cache_dir)
.with_cache_policy(Box::new(LiquidPolicy::new()))
.with_hydration_policy(Box::new(NoHydration::new()))
.with_squeeze_policy(Box::new(TranscodeEvict))
.with_eviction_policy(Box::new(TranscodeEvict))
.build(session_config)
.await?;
(v.0, Some(v.1))
Expand Down
15 changes: 4 additions & 11 deletions benchmark/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ use datafusion::{error::Result, physical_plan::ExecutionPlan};
use fastrace::Span;
use fastrace::future::FutureExt as _;
use liquid_cache::cache::CacheStats;
use liquid_cache::cache::squeeze_policies::{
Evict, SqueezePolicy, TranscodeEvict, TranscodeSqueezeEvict,
};
use liquid_cache::cache::{Evict, EvictionPolicy, TranscodeEvict};
use liquid_cache_common::rpc::ExecutionMetricsResponse;
use liquid_cache_datafusion_server::{ApiResponse, ExecutionStats};
use log::info;
Expand Down Expand Up @@ -302,15 +300,13 @@ pub enum BenchmarkMode {
Arrow,
#[default]
Liquid,
LiquidNoSqueeze,
}

impl BenchmarkMode {
pub fn to_squeeze_policy(&self) -> Box<dyn SqueezePolicy> {
pub fn to_eviction_policy(&self) -> Box<dyn EvictionPolicy> {
match self {
BenchmarkMode::Arrow => Box::new(Evict),
BenchmarkMode::Liquid => Box::new(TranscodeSqueezeEvict),
BenchmarkMode::LiquidNoSqueeze => Box::new(TranscodeEvict),
BenchmarkMode::Liquid => Box::new(TranscodeEvict),
}
}
}
Expand All @@ -323,7 +319,6 @@ impl Display for BenchmarkMode {
match self {
BenchmarkMode::Arrow => "arrow",
BenchmarkMode::Liquid => "liquid",
BenchmarkMode::LiquidNoSqueeze => "liquid-no-squeeze",
}
)
}
Expand All @@ -336,7 +331,6 @@ impl FromStr for BenchmarkMode {
Ok(match s {
"arrow" => BenchmarkMode::Arrow,
"liquid" => BenchmarkMode::Liquid,
"liquid-no-squeeze" => BenchmarkMode::LiquidNoSqueeze,
_ => return Err(format!("Invalid benchmark mode: {s}")),
})
}
Expand Down Expand Up @@ -468,10 +462,9 @@ impl Display for IterationResult {
write_border_sep(f, INNER)?;
let total_value = format!("{}", cache_stats.total_entries);
let stats_value = format!(
"(A:{} L:{} S-L:{} D-L:{} D-A:{})",
"(A:{} L:{} D-L:{} D-A:{})",
cache_stats.memory_arrow_entries,
cache_stats.memory_liquid_entries,
cache_stats.memory_squeezed_liquid_entries,
cache_stats.disk_liquid_entries,
cache_stats.disk_arrow_entries
);
Expand Down
118 changes: 0 additions & 118 deletions dev/design/03-squeeze-integer.md

This file was deleted.

32 changes: 0 additions & 32 deletions dev/design/05-predicate-cache.md

This file was deleted.

2 changes: 1 addition & 1 deletion dev/dev-tools/src/components/cache_state_view.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ pub fn CacheStateView(simulator: Signal<CacheSimulator>) -> Element {
div { class: "flex items-center gap-1 flex-wrap min-h-5",
match state.victim_status.get(&entry.entry_id) {
Some(VictimStatus::Selected) => rsx!( Badge { label: "victim".to_string(), tone: "warn", class: "".to_string() } ),
Some(VictimStatus::Squeezed) => rsx!( Badge { label: "squeezed".to_string(), tone: "neutral", class: "".to_string() } ),
Some(VictimStatus::Evicted) => rsx!( Badge { label: "evicted".to_string(), tone: "neutral", class: "".to_string() } ),
None => rsx! {},
}
if let Some(op) = state.current_operations.get(&entry.entry_id) {
Expand Down
Loading