From 28548642a13b9339ac8f32e63075e29e2d685677 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:11:42 +0000 Subject: [PATCH 1/3] refactor: handle every valid timing x lifetime state explicitly PhysicalProperties stays the struct of (ExecutionTiming, Lifetime) from or arms that list their states, and one unreachable!() arm for the invalid (IngestionTime, OneEvaluation): new, readable_at, summary_properties_allowed, run_kind, node_usage, materialized_tumbling_window_chains (now Result, the error names the node), materialized_run_rate (one classification shared by computations and the MILP's context rates) and the devtools phase_label. RunKind::Shareable and ContextKind::Shared become Materialized (kappa_m) and carry no copy of the candidate's properties. The selector and select path share cost_error and materialized_chains_of; Stage 3's window_read_through uses the shared chain predicate. Table tests over every valid state (PhysicalProperties::every_valid) for each consumer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K1M6br9PL4SDeVS1XpezKJ --- crates/cost/src/execution_demand.rs | 102 +++++++++++-- crates/cost/src/lib.rs | 8 +- crates/cost/src/usage.rs | 144 ++++++++++++------ crates/cost/tests/held_memory.rs | 2 +- crates/devtools/src/candidate_labels.rs | 76 +++++++-- crates/executor/tests/every_valid_state.rs | 107 +++++++++++++ crates/executor/tests/main.rs | 1 + .../tests/materialized_views.rs | 2 +- .../src/catalog/configurations.rs | 25 ++- .../src/catalog/realize.rs | 7 +- crates/physical-asap-optimization/src/lib.rs | 23 +-- crates/plan-selection/src/groups.rs | 84 ++++++---- crates/plan-selection/src/groups/tests.rs | 48 +++++- crates/plan-selection/src/lib.rs | 127 ++++++++++++++- crates/plan-selection/src/milp/cost_terms.rs | 32 ++-- crates/plan-selection/src/milp/decode.rs | 2 +- .../plan-selection/src/milp/lp_diagnosis.rs | 2 +- crates/plan-selection/src/milp/model.rs | 50 +++--- crates/plan-selection/src/milp/model/tests.rs | 2 +- crates/plan-selection/src/milp/warm_start.rs | 15 +- crates/plan-selection/src/report.rs | 7 + crates/plan-selection/src/select.rs | 39 ++++- crates/plan-selection/src/selector.rs | 22 +-- crates/types/src/physical/properties.rs | 137 +++++++++++++---- crates/types/tests/physical_export.rs | 57 +++++++ 25 files changed, 894 insertions(+), 227 deletions(-) create mode 100644 crates/executor/tests/every_valid_state.rs diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/execution_demand.rs index a7ddcaccf..4f8aa8d54 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/execution_demand.rs @@ -10,7 +10,7 @@ use super::analytical::AnalyticalCostError; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::ir::properties::ExecutionTiming; -use asap_types::physical::PhysicalProperties; +use asap_types::physical::{Lifetime, PhysicalProperties}; use asap_types::workload::{QueryRecurrence, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -334,6 +334,41 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap Option { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (properties.timing(), properties.lifetime()) { + (QueryTime, Lifetime::None) => None, + (IngestionTime, Lifetime::None) => None, + (IngestionTime, Lifetime::AcrossEvaluations(_)) => { + Some(MaterializedRunRate::PerSecondOfArrivingData) + } + (QueryTime, Lifetime::OneEvaluation(_)) => Some(MaterializedRunRate::PerEvaluation), + (QueryTime, Lifetime::AcrossEvaluations(_)) => Some(MaterializedRunRate::PerRequest), + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } +} + /// How often each node of `dag` is computed: `(multiplier, per_second)` /// (module doc, "Cost model"). A materialized node is computed once for all /// its readers: once per second at ingestion time (materialized at ingestion time), at @@ -359,7 +394,7 @@ pub fn computations( root_rates: Option<&[f64]>, ) -> Result, AnalyticalCostError> { let range_of = |query: usize| range_evaluations.get(query).copied().flatten(); - let mut materialized_nodes: HashMap = HashMap::new(); + let mut materialized_nodes: HashMap = HashMap::new(); for node in &dag.nodes { let properties = node.physical_properties().map_err(|reason| { AnalyticalCostError::InvalidPhysicalProperties { @@ -367,8 +402,8 @@ pub fn computations( reason, } })?; - if properties.lifetime().is_materialized() { - materialized_nodes.insert(node.id, properties); + if let Some(rate) = materialized_run_rate(properties) { + materialized_nodes.insert(node.id, rate); } } let inputs = |id: PhysicalASAPNodeId| { @@ -392,17 +427,16 @@ pub fn computations( } } let mut rate_of_materialized = HashMap::new(); - for (&id, &properties) in &materialized_nodes { - let at_ingestion = properties.timing() == ExecutionTiming::IngestionTime; - // A query-time chain materialized across evaluations is built once per request. - let per_request = !at_ingestion && properties.lifetime().is_across_evaluations(); - let rate = match at_ingestion { - true => 1.0, - _ => evaluation_rate( - reached.get(&id).into_iter().flatten().filter_map(|&q| { - let range = if per_request { None } else { range_of(q) }; - recurrence.get(q).map(|r| (r, range)) - }), + for (&id, &materialized_rate) in &materialized_nodes { + let queries = || reached.get(&id).into_iter().flatten(); + let rate = match materialized_rate { + MaterializedRunRate::PerSecondOfArrivingData => 1.0, + MaterializedRunRate::PerEvaluation => evaluation_rate( + queries().filter_map(|&q| recurrence.get(q).map(|r| (r, range_of(q)))), + non_repeating_query_horizon_seconds, + )?, + MaterializedRunRate::PerRequest => evaluation_rate( + queries().filter_map(|&q| recurrence.get(q).map(|r| (r, None))), non_repeating_query_horizon_seconds, )?, }; @@ -423,7 +457,7 @@ pub fn computations( let refresh_windows = dag.refresh_windows( materialized_nodes .iter() - .filter(|(_, properties)| properties.timing() == ExecutionTiming::IngestionTime) + .filter(|(_, &rate)| rate == MaterializedRunRate::PerSecondOfArrivingData) .map(|(&id, _)| id), ); let mut count: HashMap<_, (u64, f64)> = HashMap::new(); @@ -462,6 +496,42 @@ pub fn computations( #[cfg(test)] mod tests { use super::*; + + /// How often each valid state is computed when materialized, and `None` for + /// the two not-materialized states. + #[test] + fn kept_run_rate_over_every_valid_state() { + use asap_types::physical::Medium; + use ExecutionTiming::{IngestionTime, QueryTime}; + let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); + let mut expected = vec![ + (at(QueryTime, Lifetime::None), None), + (at(IngestionTime, Lifetime::None), None), + ]; + for medium in [Medium::Memory, Medium::Disk] { + expected.extend([ + ( + at(IngestionTime, Lifetime::AcrossEvaluations(medium)), + Some(MaterializedRunRate::PerSecondOfArrivingData), + ), + ( + at(QueryTime, Lifetime::OneEvaluation(medium)), + Some(MaterializedRunRate::PerEvaluation), + ), + ( + at(QueryTime, Lifetime::AcrossEvaluations(medium)), + Some(MaterializedRunRate::PerRequest), + ), + ]); + } + for properties in PhysicalProperties::every_valid() { + let (_, rate) = expected + .iter() + .find(|(state, _)| *state == properties) + .expect("every valid state has an expectation"); + assert_eq!(materialized_run_rate(properties), *rate, "{properties:?}"); + } + } use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; fn fixed(interval: u32, phase: Option) -> QueryRecurrence { diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index f921d3bb7..708cb7ac9 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -41,10 +41,10 @@ pub use measured::{ pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, - plan_cost, query_reach, retained_tumbling_windows, trailing_window, - tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, - NodeMap, NodeUsage, ProducerOutput, + family_name, is_materialized_tumbling_window_chain, maintained_window_ms, + materialized_tumbling_window_chains, node_map, node_usage, plan_cost, query_reach, + retained_tumbling_windows, trailing_window, tumbling_window_retention_ms, + tumbling_window_retention_ms_of_plan, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, }; /// Cost per second of wall time; one unit is one CPU-millisecond under diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 781351dd1..a31d4fada 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -26,11 +26,12 @@ use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, ScanWindow, }; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, }; use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; -use asap_types::physical::{Lifetime, Medium}; +use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; use asap_types::workload::bounds::{ @@ -532,7 +533,8 @@ pub struct NodeUsage<'a> { /// that consumer's evaluation runs ([`plan_cost`]), rather than one copy /// held until the longest query reading the node finishes). pub held: Option<(u64, f64, bool)>, - /// An in-memory ingestion-time state's bytes per window. + /// A state materialized at ingestion time: its bytes per window, the newest of + /// which each ingest worker builds in memory. pub maintained: Option, /// A query-time scan's source and the raw bytes it reads. pub raw_scan: Option<(&'a Source, u64)>, @@ -957,16 +959,20 @@ pub fn node_usage<'a>( // query time (all of them for a chain kept across // evaluations, whose scan Stage 2 bounds to its newest tumbling window, // the only one it builds). - let rows_per_tumbling_window = match (ingestion, node.lifetime) { - (true, _) => u64::try_from( + let rows_per_tumbling_window = match timing_and_lifetime(node)? { + (IngestionTime, Lifetime::None) + | (IngestionTime, Lifetime::AcrossEvaluations(_)) => u64::try_from( (u128::from(input.rows) * u128::from(*tumbling_window_width_ms)) .div_ceil(1_000), ) .map_err(|_| AnalyticalCostError::Overflow)?, - (false, Lifetime::AcrossEvaluations(_)) => input.rows, - (false, _) => input + (QueryTime, Lifetime::AcrossEvaluations(_)) => input.rows, + (QueryTime, Lifetime::None) | (QueryTime, Lifetime::OneEvaluation(_)) => input .rows .div_ceil(u64::from(*tumbling_window_count).max(1)), + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } }; let input_rows = StateInputRows { rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), @@ -1130,24 +1136,35 @@ pub fn node_usage<'a>( // latency, or a materialized consumer's longest reader's, as a // OneEvaluation state is). let state = out.bytes.max(estimate.peak_memory_bytes()); - let (materialized_bytes, medium) = match (ingestion, node.lifetime, &node.payload) { - (_, Lifetime::None, _) => { + let is_chain = matches!( + node.payload, + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ); + // A whole state kept across evaluations: it and the one being built. + let whole_state_kept = || state.checked_mul(2).ok_or(AnalyticalCostError::Overflow); + let (timing, lifetime) = timing_and_lifetime(node)?; + let (kept, medium) = match (timing, lifetime) { + // Working memory while it runs, once per consumer that recomputes it, + // and held through each such consumer's evaluation. + (QueryTime, Lifetime::None) => { let working_memory_bytes = estimate.peak_memory_bytes(); peak_memory_bytes = working_memory_bytes.saturating_mul(multiplier); - if !ingestion && working_memory_bytes > 0 { + if working_memory_bytes > 0 { held = Some((working_memory_bytes, per_second, true)); } (0, Medium::Memory) } - // A tumbling-window chain materialized across evaluations holds the tumbling windows its + // Working memory while it runs, once per ingestion run that computes + // it; no query evaluation holds it. + (IngestionTime, Lifetime::None) => { + peak_memory_bytes = estimate.peak_memory_bytes().saturating_mul(multiplier); + (0, Medium::Memory) + } + // A tumbling-window chain kept across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window // width of the readers' range-evaluation span), all but the // newest between evaluations and all while one runs. - ( - false, - Lifetime::AcrossEvaluations(medium), - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ) => { + (QueryTime, Lifetime::AcrossEvaluations(medium)) if is_chain => { let tumbling_windows = tumbling_windows_of(node); let retained = retained_tumbling_window_count; let between = (state / tumbling_windows) @@ -1172,7 +1189,8 @@ pub fn node_usage<'a>( detail += &format!("; materializes {between} bytes across evaluations"); (0, medium) } - (_, Lifetime::OneEvaluation(medium), _) => { + (QueryTime, Lifetime::AcrossEvaluations(medium)) => (whole_state_kept()?, medium), + (QueryTime, Lifetime::OneEvaluation(medium)) => { match medium { Medium::Memory => { held = Some((state, per_second, false)); @@ -1183,11 +1201,7 @@ pub fn node_usage<'a>( (0, medium) } // The retained tumbling windows and the one being built. - ( - true, - Lifetime::AcrossEvaluations(medium), - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ) => { + (IngestionTime, Lifetime::AcrossEvaluations(medium)) if is_chain => { let tumbling_windows = tumbling_windows_of(node); ( (state / tumbling_windows) @@ -1200,13 +1214,14 @@ pub fn node_usage<'a>( medium, ) } - // A whole state materialized across evaluations: it and the one being built. - (_, Lifetime::AcrossEvaluations(medium), _) => ( - state.checked_mul(2).ok_or(AnalyticalCostError::Overflow)?, - medium, - ), + (IngestionTime, Lifetime::AcrossEvaluations(medium)) => (whole_state_kept()?, medium), + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } }; - if ingestion && node.lifetime == Lifetime::AcrossEvaluations(Medium::Memory) { + // Each ingest worker builds its newest window in memory, whatever + // medium the completed windows are materialized on. + if let (IngestionTime, Lifetime::AcrossEvaluations(_)) = (timing, lifetime) { maintained = Some(state / tumbling_windows_of(node)); } if materialized_bytes > 0 { @@ -1604,22 +1619,65 @@ pub type NodeMap<'a> = /// deployment keeps query-time results and every reader builds one new /// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the /// older ones, which complete the readers' look-back, are kept from -/// earlier evaluations. -pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { - dag.nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.output_state.timing, node.lifetime), - ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ExecutionTiming::QueryTime, - Lifetime::AcrossEvaluations(_), - ) - ) - }) - .map(|node| node.id) - .collect() +/// earlier evaluations. Fails on a node whose timing and lifetime cannot +/// go together. +pub fn materialized_tumbling_window_chains( + dag: &PhysicalASAPDAG, +) -> Result, AnalyticalCostError> { + let mut chains = HashSet::new(); + for node in &dag.nodes { + if is_materialized_tumbling_window_chain(node)? { + chains.insert(node.id); + } + } + Ok(chains) +} + +/// Whether `node` is a tumbling-window chain kept across evaluations at +/// query time ([`materialized_tumbling_window_chains`]). Query time only: these are +/// the chains whose query-time scan reads just the newest tumbling window. +/// A chain kept across evaluations at ingestion time is built as data +/// arrives, so no query-time scan reads below it; plan selection's +/// retained-window options (`retained_pane_options`) and +/// [`tumbling_window_retention_ms`] cover both timings, since a chain materialized +/// at either timing retains windows. +pub fn is_materialized_tumbling_window_chain( + node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, +) -> Result { + let chain = matches!( + node.payload, + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ); + let kept_at_query_time = match timing_and_lifetime(node)? { + (QueryTime, Lifetime::AcrossEvaluations(_)) => true, + // Rebuilt whole by each evaluation that reads it: its scan reads + // the whole window. + (QueryTime, Lifetime::None) => false, + // Materialized only within one evaluation, which builds it whole. + (QueryTime, Lifetime::OneEvaluation(_)) => false, + // Built inside an ingestion run: no query-time scan reads below it. + (IngestionTime, Lifetime::None) => false, + // Built as data arrives: no query-time scan reads below it. + (IngestionTime, Lifetime::AcrossEvaluations(_)) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; + Ok(chain && kept_at_query_time) +} + +/// `node`'s timing and lifetime, checked by [`PhysicalProperties::new`]: +/// the error names the node when they cannot go together. +fn timing_and_lifetime( + node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, +) -> Result<(ExecutionTiming, Lifetime), AnalyticalCostError> { + let properties: PhysicalProperties = node.physical_properties().map_err(|reason| { + AnalyticalCostError::InvalidPhysicalProperties { + node: node.id, + reason, + } + })?; + Ok((properties.timing(), properties.lifetime())) } /// The widest window, in milliseconds, read above the scan `scan` at query diff --git a/crates/cost/tests/held_memory.rs b/crates/cost/tests/held_memory.rs index 079b7ea16..0e2363fde 100644 --- a/crates/cost/tests/held_memory.rs +++ b/crates/cost/tests/held_memory.rs @@ -56,7 +56,7 @@ fn cost(dag: &PhysicalASAPDAG, cost_per_memory_byte_second: f64) -> PlanCost { plan_cost( dag, &nodes, - &materialized_tumbling_window_chains(dag), + &materialized_tumbling_window_chains(dag).unwrap(), &recurrence(), &[], &data(), diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index cf2f82035..e18fd3ce6 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -14,8 +14,9 @@ use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::NonASAPOp; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::Lifetime; -use asap_types::physical::PhysicalWorkloadPlan; +use asap_types::physical::{ + InvalidPhysicalProperties, Lifetime, PhysicalProperties, PhysicalWorkloadPlan, +}; use asap_types::stages::HowToComputeAggregateMeasure; use crate::planning_stages::PlanningStagesRun; @@ -147,13 +148,8 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String let mut parts: Vec<(&str, Vec)> = Vec::new(); for node in &candidate.dag.nodes { let id = &node.id; - let phase = match (node.output_state.timing, node.lifetime) { - (_, Lifetime::None) => continue, - (ExecutionTiming::IngestionTime, _) => "ingestion time", - (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => "query time, shared", - (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => { - "query time, materialized across evaluations" - } + let Some(phase) = phase_label(node.physical_properties()) else { + continue; }; let label = match &node.payload { // Name the chain: chains of one family and tumbling-window count differ by @@ -187,6 +183,29 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String .join(" · ") } +/// The label of the phase a node with `properties` is materialized in; +/// `None` when it is not materialized. A node whose timing and lifetime +/// cannot go together is shown as such rather than dropped. +fn phase_label( + properties: Result, +) -> Option<&'static str> { + use ExecutionTiming::{IngestionTime, QueryTime}; + let Ok(properties) = properties else { + return Some("invalid timing and lifetime"); + }; + match (properties.timing(), properties.lifetime()) { + // Not materialized: no phase holds it; it runs inside its reader. + (QueryTime, Lifetime::None) => None, + (IngestionTime, Lifetime::None) => None, + (IngestionTime, Lifetime::AcrossEvaluations(_)) => Some("ingestion time"), + (QueryTime, Lifetime::OneEvaluation(_)) => Some("query time, shared"), + (QueryTime, Lifetime::AcrossEvaluations(_)) => Some("query time, kept"), + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } +} + /// A summary build, e.g. "Kll ×5 tumbling windows" or "Sum acc". fn build_label(payload: &Operator) -> String { match payload { @@ -539,6 +558,45 @@ mod tests { ); } + /// The phase label over every valid state, and an invalid pair. + #[test] + fn phase_label_over_every_valid_state() { + use asap_types::physical::Medium; + use ExecutionTiming::{IngestionTime, QueryTime}; + let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); + let mut expected = vec![ + (at(QueryTime, Lifetime::None), None), + (at(IngestionTime, Lifetime::None), None), + ]; + for medium in [Medium::Memory, Medium::Disk] { + expected.extend([ + ( + at(IngestionTime, Lifetime::AcrossEvaluations(medium)), + Some("ingestion time"), + ), + ( + at(QueryTime, Lifetime::OneEvaluation(medium)), + Some("query time, shared"), + ), + ( + at(QueryTime, Lifetime::AcrossEvaluations(medium)), + Some("query time, kept"), + ), + ]); + } + for properties in PhysicalProperties::every_valid() { + let (_, label) = expected + .iter() + .find(|(state, _)| *state == properties) + .expect("every valid state has an expectation"); + assert_eq!(phase_label(Ok(properties)), *label, "{properties:?}"); + } + assert_eq!( + phase_label(Err(InvalidPhysicalProperties::OneEvaluationAtIngestion)), + Some("invalid timing and lifetime") + ); + } + /// Raw logical operators have an exact label, rather than an empty choice label. #[test] fn connected_labels_name_exact_results() { diff --git a/crates/executor/tests/every_valid_state.rs b/crates/executor/tests/every_valid_state.rs new file mode 100644 index 000000000..17f430664 --- /dev/null +++ b/crates/executor/tests/every_valid_state.rs @@ -0,0 +1,107 @@ +//! The executor's physical planner over every valid timing × lifetime +//! state of one summary build. +use std::collections::HashMap; +use std::rc::Rc; +use std::time::Duration; + +use planner_types::ir::operator::operator_properties::{Reduction, Source}; +use planner_types::ir::physical_export::compile_physical_asap_dag; +use planner_types::ir::properties::ExecutionTiming; +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::{ + DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, +}; +use planner_types::ir::{ + apply_physical_properties, ASAPOp, NonASAPOp, Operator, OperatorNode, + PhysicalPropertiesAssignment, TimeRangeKind, +}; +use planner_types::physical::{Lifetime, PhysicalProperties}; + +/// `FinalizeExactAccumulator(SummaryAgg(Sum)(TimeRange 5 min(Scan latency_ms)))`. +fn plan() -> Rc { + let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency_ms".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + })) + .unwrap(); + let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: range, + family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::Named("value".into())), + reduction: Reduction::by(vec![]), + grouping: Default::default(), + filter: None, + })) + .unwrap(); + OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: state, + })) + .unwrap() +} + +/// What the physical planner makes of the build in each valid state: a +/// state materialized across evaluations (either timing, either medium) is +/// a materialized view slot the deployment binds; every other state is +/// computed in the query run (a `OneEvaluation` result once for the run's +/// readers). A summary build not materialized at ingestion time is not +/// built by rule: the timing pass runs an unmaterialized build at query +/// time. +#[test] +fn physical_planner_over_every_valid_state() { + use ExecutionTiming::{IngestionTime, QueryTime}; + for properties in PhysicalProperties::every_valid() { + let root = plan(); + let mut assignment = PhysicalPropertiesAssignment::default(); + let unmaterialized_at_ingestion = + (properties.timing(), properties.lifetime()) == (IngestionTime, Lifetime::None); + if !unmaterialized_at_ingestion { + assignment.set(root.children()[0], properties); + } + let timed = apply_physical_properties(&root, &assignment).unwrap(); + let mut dag = compile_physical_asap_dag(&root, &timed).unwrap(); + let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); + let compiled = asap_executor::physical_planner::compile(&dag, &views, &[3]) + .unwrap_or_else(|error| panic!("{properties:?}: {error}")); + let build = compiled.operator_name(2); + let computed_in_the_query_run = || { + assert_ne!(build, Some("MaterializedView"), "{properties:?}"); + assert!(build.is_some(), "{properties:?}"); + }; + match (properties.timing(), properties.lifetime()) { + (IngestionTime, Lifetime::AcrossEvaluations(_)) => { + assert_eq!(build, Some("MaterializedView"), "{properties:?}"); + } + (QueryTime, Lifetime::AcrossEvaluations(_)) => { + assert_eq!(build, Some("MaterializedView"), "{properties:?}"); + } + (QueryTime, Lifetime::None) => computed_in_the_query_run(), + (QueryTime, Lifetime::OneEvaluation(_)) => computed_in_the_query_run(), + (IngestionTime, Lifetime::None) => { + // Left unassigned: an unmaterialized build lands at query time. + assert_eq!( + dag.nodes[2].physical_properties(), + Ok(PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME) + ); + } + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + } +} diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index 89abe7da8..0e1ee1792 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -9,6 +9,7 @@ mod current_series_heap; mod current_timestamp; mod deployment; mod deployment_computation; +mod every_valid_state; mod exact_state_columns; mod hydra_cms_execution; mod keyed_summary_row_weight; diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index 60eddd576..a4c5dfdef 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -48,7 +48,7 @@ fn repeating_with(queries: &[&str], extra: &str) -> PlanningWorkload { /// evaluations reading its newest tumbling window, as `asap-plan-selection` prices /// it, the longest per source. fn stage3_raw_lookback_ms(dag: &PhysicalASAPDAG) -> Vec<(Source, Option)> { - let materialized_chains = asap_cost::materialized_tumbling_window_chains(dag); + let materialized_chains = asap_cost::materialized_tumbling_window_chains(dag).unwrap(); let mut lookback: Vec<(Source, Option)> = Vec::new(); for node in &dag.nodes { let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs index ef47a6475..d78777e8f 100644 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ b/crates/physical-asap-optimization/src/catalog/configurations.rs @@ -137,6 +137,22 @@ pub fn enumerate_candidate_configurations( .enumerate() .filter_map(|(query, reached)| reached.then_some(query)) .collect::>(); + // The valid states Stage 2 does not offer, by rule (every consumer + // still handles them when given one): + // - any `Disk` medium (#1217): the executor has no disk store, so a + // `Disk` state given to it executes in memory (a view slot bound to + // the deployment's in-memory store, a one-evaluation result kept + // within the query run), while cost prices it as disk; + // - (IngestionTime, None) for a summary build: an unmaterialized build + // runs at query time (`apply_physical_properties`); + // - (IngestionTime, AcrossEvaluations) for a non-summary node: rows + // are not maintained at ingestion time, only summary states; + // - (QueryTime, AcrossEvaluations) for anything but a tumbling-window + // chain: only a chain reuses what earlier evaluations built (one + // new tumbling window per evaluation); any other result is rebuilt + // whole each evaluation; + // - any ingestion-time state for `SummaryEstimate`, + // `EvaluatePopulation` and `MaintainPopulation` (`forced_query`). let mut alternatives = vec![PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME]; if summary { alternatives.push(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION); @@ -394,12 +410,19 @@ fn summary_properties_allowed( candidate_properties.timing(), candidate_properties.lifetime(), ) { - (_, Lifetime::None | Lifetime::OneEvaluation(_)) => true, + (ExecutionTiming::QueryTime, Lifetime::None) + | (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => true, + // Runs inside the ingestion refresh of the state that reads it, so + // the reader must allow ingestion, as for a build materialized there. + (ExecutionTiming::IngestionTime, Lifetime::None) => eligibility.at_ingestion.is_ok(), (ExecutionTiming::IngestionTime, Lifetime::AcrossEvaluations(_)) => { eligibility.at_ingestion.is_ok() } (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => { eligibility.may_keep_across_evaluations } + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } } } diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index e81b8efdb..1ab1b6604 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -66,10 +66,11 @@ pub fn realize_selected_candidates( seen[node] = true; assignments[node] }); - // The option materializing it as requested; option `None` (not materialized) - // for a requested lifetime `None`, at either timing. + // The option with the requested properties; option `None` (no + // entry: not materialized, placed by the timing pass) for a + // requested lifetime `None`, at either timing. let chosen = options.iter().position(|option| match option { - Some(materialized) => *materialized == requested, + Some(entry) => *entry == requested, None => !requested.lifetime().is_materialized(), }); if chosen.is_none() { diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 34a5ccfcb..87cb0d1bc 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -349,10 +349,15 @@ pub(crate) struct MaterializedOutputCandidate { } impl MaterializedOutputCandidate { - /// Its options, not materialized first: materialized at ingestion time when eligible, - /// `OneEvaluation` when two or more queries read it, and - /// query-time `AcrossEvaluations` when it may be materialized across - /// evaluations. + /// Its options, each the assignment entry it would get: `None` first + /// (no entry: not materialized, its timing left to the timing pass, + /// which places it where its consumer runs), then materialized at + /// ingestion time when eligible, `OneEvaluation` when two or more + /// queries read it, and query-time `AcrossEvaluations` when it may be + /// kept across evaluations. Every materialized option is in memory: + /// Stage 2 offers no `Disk` state, since the executor runs a `Disk` + /// state in memory (#1217). The other valid states it does not offer + /// are listed at `enumerate_candidate_configurations`. pub(crate) fn options(&self) -> Vec> { let mut options = vec![None]; if self.at_ingestion.is_ok() { @@ -522,11 +527,11 @@ fn assigned_candidate( assignment: Vec>, ) -> Result { let mut options = PhysicalPropertiesAssignment::default(); - for (materialized_output_candidate, materialized) in + for (materialized_output_candidate, entry) in materialized_output_candidates.iter().zip(&assignment) { - if let Some(materialized) = *materialized { - options.set(&materialized_output_candidate.node, materialized); + if let Some(properties) = *entry { + options.set(&materialized_output_candidate.node, properties); } } let operators: Vec> = implemented @@ -933,8 +938,8 @@ fn down_closed( below: &[Vec], assignment: &[Option], ) -> bool { - let at_ingestion = |materialized: &Option| { - materialized.is_some_and(|p| p.timing() == ExecutionTiming::IngestionTime) + let at_ingestion = |entry: &Option| { + entry.is_some_and(|p| p.timing() == ExecutionTiming::IngestionTime) }; assignment.iter().enumerate().all(|(c, m)| { !at_ingestion(m) diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs index 60122bbd5..26ab74444 100644 --- a/crates/plan-selection/src/groups.rs +++ b/crates/plan-selection/src/groups.rs @@ -337,17 +337,23 @@ pub(crate) enum RunKind { /// arriving data for all of them. Ingestion, /// Materialized (lifetime `OneEvaluation` or `AcrossEvaluations`, at either - /// timing): selected once, its own execution context for its inputs, + /// timing): one `x_v`, its own execution context `κ_m` for its inputs, /// charged once. - Shareable, + Materialized, } /// The [`RunKind`] of a candidate with `properties`. pub(crate) fn run_kind(properties: PhysicalProperties) -> RunKind { - match (properties.lifetime(), properties.timing()) { - (Lifetime::None, ExecutionTiming::QueryTime) => RunKind::PerContext, - (Lifetime::None, ExecutionTiming::IngestionTime) => RunKind::Ingestion, - (Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), _) => RunKind::Shareable, + use ExecutionTiming::{IngestionTime, QueryTime}; + match (properties.timing(), properties.lifetime()) { + (QueryTime, Lifetime::None) => RunKind::PerContext, + (IngestionTime, Lifetime::None) => RunKind::Ingestion, + (QueryTime, Lifetime::OneEvaluation(_)) => RunKind::Materialized, + (QueryTime, Lifetime::AcrossEvaluations(_)) => RunKind::Materialized, + (IngestionTime, Lifetime::AcrossEvaluations(_)) => RunKind::Materialized, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } } } @@ -451,7 +457,7 @@ impl CandidateLinks { .into_iter() .filter(|producer| { producer.0 != reader - && (kind[reader] == RunKind::Shareable + && (kind[reader] == RunKind::Materialized || slot[producer.0] != slot[reader]) }) .collect() @@ -467,7 +473,7 @@ impl CandidateLinks { /// Whether `candidate` is materialized: selected once, its own context. fn is_shareable(&self, candidate: PhysicalCandidateId) -> bool { - self.kind[candidate.0] == RunKind::Shareable + self.kind[candidate.0] == RunKind::Materialized } } @@ -674,13 +680,17 @@ fn queries_reaching( reaching } -/// Which execution context (#1095 §2.0): a query evaluation, an ingestion -/// pipeline `I_r`, or a shareable candidate's own inputs. +/// Which execution context κ (#1095 §2.0, #1204): a query evaluation +/// `κ_q`, an ingestion pipeline `I_r`, or a materialized candidate's +/// computation `κ_m`, which runs the not-materialized candidates it reads. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub(crate) enum ContextKind { + /// `κ_q`: query `q`'s evaluation. Query(usize), + /// `I_r`: the ingestion pipeline of refresh window `r`. Ingestion(IngestionPipeline), - Shared(PhysicalCandidateId), + /// `κ_m`: the computation of materialized candidate `m`. + Materialized(PhysicalCandidateId), } /// Which ingestion pipeline `I_r` (#1095 §2.5 O3(b)): the deployment @@ -813,7 +823,7 @@ impl ExecutionContext { match kind { RunKind::PerContext => self.at_query_time, RunKind::Ingestion => matches!(self.kind, ContextKind::Ingestion(_)), - RunKind::Shareable => false, + RunKind::Materialized => false, } } } @@ -825,7 +835,7 @@ impl MilpInputs { pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { match self.contexts[context].kind { ContextKind::Ingestion(_) => Some(context), - ContextKind::Shared(m) => self.ingestion_context_of_shareable.get(&m).copied(), + ContextKind::Materialized(m) => self.ingestion_context_of_shareable.get(&m).copied(), ContextKind::Query(_) => None, } } @@ -917,14 +927,14 @@ pub(crate) fn execution_contexts( // Only an ingestion-time shareable candidate reads a candidate // of an `I_r` (Stage 2's timing rule). let pipeline = match context.kind { - ContextKind::Shared(m) if !context.at_query_time => { + ContextKind::Materialized(m) if !context.at_query_time => { Some(ingestion_pipeline(m, refresh_windows)) } _ => None, }; for &class in &context.options { match kind[class.0] { - RunKind::Shareable => { + RunKind::Materialized => { if !context_of_shareable.contains_key(&class) && !pending.contains(&class) { pending.push(class); } @@ -961,7 +971,7 @@ pub(crate) fn execution_contexts( let at_query_time = physical.candidates[class.0].properties.timing() == ExecutionTiming::QueryTime; let mut context = ExecutionContext { - kind: ContextKind::Shared(class), + kind: ContextKind::Materialized(class), at_query_time, options: Vec::new(), }; @@ -989,7 +999,7 @@ pub(crate) fn execution_contexts( } let mut ingestion_context_of_shareable = BTreeMap::new(); for context in shared { - if let ContextKind::Shared(class) = context.kind { + if let ContextKind::Materialized(class) = context.kind { context_of_shareable.insert(class, contexts.len()); if !context.at_query_time { let pipeline = ingestion_pipeline(class, refresh_windows); @@ -1080,7 +1090,9 @@ fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { /// plan where `path` is the scan's only reader: the shifts' offsets add up, /// and a `TimeRange` reads its range, a kept pane chain one pane, any other /// pane chain its whole window; any other reader reads `Unbounded`. -fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { +fn window_read_through( + path: &[&PhysicalASAPDAGNode], +) -> Result { let mut end_offset_ms: i64 = 0; for node in path { let width_ms: u128 = match &node.payload { @@ -1092,9 +1104,7 @@ fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - }) if node.output_state.timing == ExecutionTiming::QueryTime - && node.lifetime.is_across_evaluations() => - { + }) if asap_cost::is_materialized_tumbling_window_chain(node)? => { u128::from(*tumbling_window_width_ms) } Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { @@ -1102,14 +1112,14 @@ fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { tumbling_window_count, .. }) => u128::from(*tumbling_window_width_ms) * u128::from(*tumbling_window_count), - _ => return ScanWindow::Unbounded, + _ => return Ok(ScanWindow::Unbounded), }; - return ScanWindow::Within { + return Ok(ScanWindow::Within { end_offset_ms, width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; + }); } - ScanWindow::Unbounded + Ok(ScanWindow::Unbounded) } /// The windows query-time scan `scan` may be read at: each reader's @@ -1147,7 +1157,7 @@ fn scan_window_options( window_union(read.iter().copied()) }) }; - let shareable = kind[scan.0] == RunKind::Shareable; + let shareable = kind[scan.0] == RunKind::Materialized; let mut all_kinds: Vec = Vec::new(); for context in contexts { if !context.options.contains(&scan) { @@ -1168,7 +1178,7 @@ fn scan_window_options( } // A shareable candidate reading the scan as its own input reads it // at its own window; a read as an output reads it whole. - if let ContextKind::Shared(own) = context.kind { + if let ContextKind::Materialized(own) = context.kind { if producers_per_input[own.0] .iter() .any(|producers| producers.contains(&scan)) @@ -1181,7 +1191,7 @@ fn scan_window_options( } let read_whole = match context.kind { ContextKind::Query(_) => output_candidates.iter().any(|out| out.contains(&scan)), - ContextKind::Shared(_) | ContextKind::Ingestion(_) => false, + ContextKind::Materialized(_) | ContextKind::Ingestion(_) => false, }; if read_whole && !kinds.contains(&ScanWindow::Unbounded) { kinds.push(ScanWindow::Unbounded); @@ -1256,7 +1266,8 @@ fn scan_windows_per_reader( scan: PhysicalCandidateId, nodes: &[PhysicalASAPDAGNode], readers: &[Vec<(PhysicalCandidateId, usize)>], -) -> Vec<(PhysicalCandidateId, Vec)> { +) -> Result)>, asap_cost::analytical::AnalyticalCostError> +{ /// Time shifts nested deeper than this are not followed. const MAX_SHIFTS: usize = 4; let mut per_reader: Vec<(PhysicalCandidateId, Vec)> = Vec::new(); @@ -1272,7 +1283,7 @@ fn scan_windows_per_reader( if !follow { // The path ends at a reader: what the scan reads under it. let through: Vec<_> = path[1..].iter().map(|class| &nodes[class.0]).collect(); - let window = window_read_through(&through); + let window = window_read_through(&through)?; let Some(&reader) = path.get(1) else { continue; }; @@ -1292,7 +1303,7 @@ fn scan_windows_per_reader( } } per_reader.sort_by_key(|(reader, _)| *reader); - per_reader + Ok(per_reader) } /// The panes kept pane chain `chain` may keep, one per query that can reach @@ -1567,8 +1578,15 @@ fn candidate_data( } let windows = match (&node.payload, node.output_state.timing) { (Operator::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - result[class.0].windows_per_reader = - scan_windows_per_reader(class, &nodes, &readers); + match scan_windows_per_reader(class, &nodes, &readers) { + Ok(windows) => result[class.0].windows_per_reader = windows, + Err(error) => { + let reason = error.to_string(); + outputs[class.0] = Some(Err(reason.clone())); + result[class.0].usage = Err(reason); + continue; + } + } scan_window_options( class, &result[class.0].windows_per_reader, diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs index 530ccda88..718ce32db 100644 --- a/crates/plan-selection/src/groups/tests.rs +++ b/crates/plan-selection/src/groups/tests.rs @@ -525,12 +525,12 @@ fn execution_contexts_on_a_hand_built_catalog() { (ContextKind::Query(0), vec![middle, top_0]), (ContextKind::Query(1), vec![middle, top_1]), (ContextKind::Query(2), vec![lone]), - (ContextKind::Shared(middle), vec![leaf]), + (ContextKind::Materialized(middle), vec![leaf]), ] ); assert_eq!(inputs.context_of_shareable, BTreeMap::from([(middle, 3)])); assert!(inputs.contexts[3].computes(RunKind::PerContext)); - assert!(!inputs.contexts[3].computes(RunKind::Shareable)); + assert!(!inputs.contexts[3].computes(RunKind::Materialized)); assert_eq!( inputs.output_candidates, vec![vec![top_0], vec![top_1], vec![lone]] @@ -649,7 +649,10 @@ fn two_queries_share_a_pane_chain_through_the_real_pipeline() { assert!(!shared_chains.is_empty()); for &chain in &shared_chains { let context = inputs.context_of_shareable[&chain]; - assert_eq!(inputs.contexts[context].kind, ContextKind::Shared(chain)); + assert_eq!( + inputs.contexts[context].kind, + ContextKind::Materialized(chain) + ); } // Every context computes only candidates of its own kind and reads the // others across its boundary. @@ -658,7 +661,7 @@ fn two_queries_share_a_pane_chain_through_the_real_pipeline() { let kind = inputs.kind[option.0]; assert!( context.computes(kind) - || kind == RunKind::Shareable + || kind == RunKind::Materialized || (kind == RunKind::Ingestion && !matches!(context.kind, ContextKind::Ingestion(_))) ); @@ -1089,3 +1092,40 @@ fn producers_of_an_input_read_one_view_window() { let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); assert!(producer_window_disagreements_of_all(&physical).is_empty()); } + +/// `run_kind` over every valid state: not materialized runs per context at +/// query time and in the ingestion pipeline at ingestion time; every +/// materialized state, any medium, is `Materialized`. +#[test] +fn run_kind_over_every_valid_state() { + use asap_types::physical::{Lifetime, Medium}; + use ExecutionTiming::{IngestionTime, QueryTime}; + let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); + let mut expected = vec![ + (at(QueryTime, Lifetime::None), RunKind::PerContext), + (at(IngestionTime, Lifetime::None), RunKind::Ingestion), + ]; + for medium in [Medium::Memory, Medium::Disk] { + expected.extend([ + ( + at(QueryTime, Lifetime::OneEvaluation(medium)), + RunKind::Materialized, + ), + ( + at(QueryTime, Lifetime::AcrossEvaluations(medium)), + RunKind::Materialized, + ), + ( + at(IngestionTime, Lifetime::AcrossEvaluations(medium)), + RunKind::Materialized, + ), + ]); + } + for properties in PhysicalProperties::every_valid() { + let (_, kind) = expected + .iter() + .find(|(state, _)| *state == properties) + .expect("every valid state has an expectation"); + assert_eq!(run_kind(properties), *kind, "{properties:?}"); + } +} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 81c97e04d..18e72d90b 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -396,7 +396,7 @@ mod tests { self_calibrated_cost( dag, &node_map(dag), - &materialized_tumbling_window_chains(dag), + &materialized_tumbling_window_chains(dag).expect("valid physical properties"), recurrence, &[], data, @@ -5047,6 +5047,127 @@ mod tests { assert!(cost.usage.average_memory_bytes < (materialized_bytes + bytes) as f64); } + /// The KLL DAG whose node has `properties`: the build itself, except + /// for (IngestionTime, None), which is the scan below the build + /// materialized at ingestion time (a summary build that is not + /// materialized runs at query time). + fn kll_dag_with(properties: PhysicalProperties) -> PhysicalASAPDAG { + match (properties.timing(), properties.lifetime()) { + (ExecutionTiming::IngestionTime, Lifetime::None) => kll_dag(AT_INGESTION), + _ => kll_dag(Some(properties)), + } + } + + /// Cost over every valid state of the KLL build: each prices without + /// error; a state materialized on disk adds disk bytes and one in memory + /// adds none. + #[test] + fn cost_over_every_valid_state() { + let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); + let not_materialized = price(&kll_dag(None), &recurrences(1), &data(), &settings).unwrap(); + for properties in PhysicalProperties::every_valid() { + let dag = kll_dag_with(properties); + assert!( + dag.nodes + .iter() + .any(|node| node.physical_properties() == Ok(properties)), + "{properties:?}" + ); + let cost = price(&dag, &recurrences(1), &data(), &settings) + .unwrap_or_else(|error| panic!("{properties:?}: {error:?}")); + assert!(cost.cost_per_second.unwrap().is_finite(), "{properties:?}"); + let disk_added = cost.usage.disk_bytes > not_materialized.usage.disk_bytes; + let on_disk = match properties.lifetime() { + Lifetime::None => false, + Lifetime::OneEvaluation(medium) | Lifetime::AcrossEvaluations(medium) => { + medium == Medium::Disk + } + }; + assert_eq!(disk_added, on_disk, "{properties:?}"); + } + } + + /// Execution demand (`asap_cost::execution_demand::computations`) over + /// every valid state of the KLL build, read by one query every minute: + /// materialized at ingestion time, once per second; materialized at query + /// time (one evaluation, or across evaluations, built once per request), + /// at the query's rate; not materialized at query time, once per consumer + /// at its rate; not materialized at ingestion time (the scan below the + /// ingestion-time build), once per second in its ingestion run. + #[test] + fn computations_over_every_valid_state() { + use asap_cost::execution_demand::computations; + use ExecutionTiming::{IngestionTime, QueryTime}; + let per_minute = 1.0 / 60.0; + for properties in PhysicalProperties::every_valid() { + let dag = kll_dag_with(properties); + let times = computations(&dag, &[repeating(MINUTE)], &[], 3600.0, None) + .unwrap_or_else(|error| panic!("{properties:?}: {error}")); + let node = dag + .nodes + .iter() + .find(|node| node.physical_properties() == Ok(properties)) + .unwrap_or_else(|| panic!("{properties:?}")); + let (_, rate) = times[&node.id]; + let expected = match (properties.timing(), properties.lifetime()) { + (IngestionTime, Lifetime::None) => 1.0, + (IngestionTime, Lifetime::AcrossEvaluations(_)) => 1.0, + (QueryTime, Lifetime::None) => per_minute, + (QueryTime, Lifetime::OneEvaluation(_)) => per_minute, + (QueryTime, Lifetime::AcrossEvaluations(_)) => per_minute, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; + assert!( + close(rate, expected), + "{properties:?}: {rate} vs {expected}" + ); + } + } + + /// The partial copies of the newest window over every valid state of + /// B1's chain: each ingest worker beyond the first holds one for a chain + /// materialized at ingestion time, in memory or on disk (the newest + /// window is built in memory whatever the medium); no other state has + /// them. + #[test] + fn partial_copies_over_every_valid_state() { + use ExecutionTiming::{IngestionTime, QueryTime}; + let rows_per_s = 1e9; + let b1 = tumbling_window_candidates(rows_per_s).remove(1); + let chain = node_of(&b1.dag, is_chain); + for properties in PhysicalProperties::every_valid() { + let mut dag = b1.dag.clone(); + let node = dag.nodes.iter_mut().find(|n| n.id == chain).unwrap(); + node.output_state.timing = properties.timing(); + node.lifetime = properties.lifetime(); + let cost = price( + &dag, + &[repeating(MINUTE)], + &arriving(rows_per_s), + &CostInputs::default(), + ) + .unwrap_or_else(|error| panic!("{properties:?}: {error:?}")); + let copies = cost.per_node[&chain].detail.contains("partial copies"); + let expected = match (properties.timing(), properties.lifetime()) { + (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::None) => false, + (QueryTime, Lifetime::None) => false, + (QueryTime, Lifetime::OneEvaluation(_)) => false, + (QueryTime, Lifetime::AcrossEvaluations(_)) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; + assert_eq!( + copies, expected, + "{properties:?}: {}", + cost.per_node[&chain].detail + ); + } + } + /// The KLL build of a `quantile_over_time` candidate, with its sketch /// parameters, and the candidate. fn kll_build() -> (PhysicalWorkloadPlan, PhysicalASAPNodeId, SketchParams) { @@ -5362,7 +5483,7 @@ mod tests { plan_cost( &candidate.dag, &node_map(&candidate.dag), - &materialized_tumbling_window_chains(&candidate.dag), + &materialized_tumbling_window_chains(&candidate.dag).unwrap(), &[repeating(MINUTE)], &[], &arriving(10_000.0), @@ -5593,7 +5714,7 @@ mod tests { let cost = self_calibrated_cost( &candidate.dag, &node_map(&candidate.dag), - &materialized_tumbling_window_chains(&candidate.dag), + &materialized_tumbling_window_chains(&candidate.dag).unwrap(), std::slice::from_ref(&recurrence), &[range], &data(), diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs index 589a60bfd..842dec882 100644 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ b/crates/plan-selection/src/milp/cost_terms.rs @@ -52,7 +52,9 @@ //! [`SelectionMilp::inexact`] by reason. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use asap_cost::execution_demand::{evaluation_rate, schedule_groups}; +use asap_cost::execution_demand::{ + evaluation_rate, materialized_run_rate, schedule_groups, MaterializedRunRate, +}; use asap_cost::CostModel; use asap_types::ir::physical_export::ScanWindow; use asap_types::ir::properties::ExecutionTiming; @@ -212,7 +214,7 @@ fn producer_gives( // `I_r`. let producer_run = match milp.kind(producer) { _ if milp.computes(context, producer) => context, - RunKind::Shareable => milp.context_of(producer)?, + RunKind::Materialized => milp.context_of(producer)?, RunKind::Ingestion => milp.ingestion_context_read_by(context)?, RunKind::PerContext => return None, }; @@ -490,7 +492,7 @@ fn window_terms( } // Each read: (reading context, window kind, reader's indicator). let mut reads: Vec<(usize, ScanWindow, Expression)> = Vec::new(); - let reading_contexts: Vec = if milp.kind(scan) == RunKind::Shareable { + let reading_contexts: Vec = if milp.kind(scan) == RunKind::Materialized { (0..milp.choice.len()) .filter(|&other| milp.choice[other].contains_key(&scan)) .collect() @@ -516,7 +518,7 @@ fn window_terms( } Reader::Output(_) => (ScanWindow::Unbounded, Expression::from(1.0)), Reader::OwnInput(_) => { - let ContextKind::Shared(own) = milp.inputs.contexts[reading].kind else { + let ContextKind::Materialized(own) = milp.inputs.contexts[reading].kind else { continue; }; // `m` itself reads the scan: at its own window. @@ -842,15 +844,21 @@ fn context_rates( let rate = match milp.inputs.contexts[context].kind { ContextKind::Query(query) => ContextRate::Fixed(query_rate[query]), ContextKind::Ingestion(_) => ContextRate::Fixed(1.0), - ContextKind::Shared(m) => { + ContextKind::Materialized(m) => { let properties = milp.physical.candidates[m.0].properties; - let per_request = match (properties.timing(), properties.lifetime()) { - (ExecutionTiming::IngestionTime, _) | (_, Lifetime::None) => { + let per_request = match materialized_run_rate(properties) { + Some(MaterializedRunRate::PerSecondOfArrivingData) => { rates.push(ContextRate::Fixed(1.0)); continue; } - (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => false, - (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => true, + Some(MaterializedRunRate::PerEvaluation) => false, + Some(MaterializedRunRate::PerRequest) => true, + // `κ_m` exists only for a materialized `m` (`RunKind::Materialized`). + None => { + return Err(SelectionMilpError::Cost(format!( + "context of candidate {m:?}, which is not materialized ({properties:?})" + ))) + } }; let readers = milp.queries_reaching(m); let demand = |query: usize| { @@ -996,7 +1004,7 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec candidate_data.properties.timing(), candidate_data.properties.lifetime(), ); - let own = milp.inputs.contexts[context].kind == ContextKind::Shared(candidate); + let own = milp.inputs.contexts[context].kind == ContextKind::Materialized(candidate); // The most this run adds to its context's execution cost: all its // terms at 1. let mut run_execution = 0.0; @@ -1040,7 +1048,7 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec ); charge(&held.into(), per_byte_ms * query_rate[query]); } - ContextKind::Shared(m) => { + ContextKind::Materialized(m) => { if own && lifetime != Lifetime::OneEvaluation(Medium::Memory) { milp.note_inexact("held bytes of a shareable candidate not in memory"); } @@ -1119,7 +1127,7 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec let ContextRate::Components(components) = &rates[context] else { continue; }; - let ContextKind::Shared(m) = milp.inputs.contexts[context].kind else { + let ContextKind::Materialized(m) = milp.inputs.contexts[context].kind else { continue; }; let region = milp diff --git a/crates/plan-selection/src/milp/decode.rs b/crates/plan-selection/src/milp/decode.rs index c3dddfd60..f764d66e7 100644 --- a/crates/plan-selection/src/milp/decode.rs +++ b/crates/plan-selection/src/milp/decode.rs @@ -66,7 +66,7 @@ pub(crate) fn decode( return Some(Run::Computed(context, producer)); } match kind { - RunKind::Shareable => chosen + RunKind::Materialized => chosen .selected .contains(&producer) .then_some(Run::Shared(producer)), diff --git a/crates/plan-selection/src/milp/lp_diagnosis.rs b/crates/plan-selection/src/milp/lp_diagnosis.rs index 5b0b6e5b0..1913fc51f 100644 --- a/crates/plan-selection/src/milp/lp_diagnosis.rs +++ b/crates/plan-selection/src/milp/lp_diagnosis.rs @@ -192,7 +192,7 @@ impl Diagnosis { } crate::groups::IngestionPipeline::Alone(m) => format!("I[alone m{}]", m.0), }, - crate::groups::ContextKind::Shared(m) => format!("m{}", m.0), + crate::groups::ContextKind::Materialized(m) => format!("m{}", m.0), }; for (&c, variable) in choices { labels.insert(column_of[variable], format!("p[{place}] {}", candidate(c))); diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 65ada5529..479bd939a 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -367,7 +367,7 @@ impl<'a> SelectionMilp<'a> { context: usize, candidate: PhysicalCandidateId, ) -> Option { - if self.inputs.contexts[context].kind == ContextKind::Shared(candidate) { + if self.inputs.contexts[context].kind == ContextKind::Materialized(candidate) { return self.selected.get(&candidate).copied(); } if !self.computes(context, candidate) { @@ -404,7 +404,7 @@ impl<'a> SelectionMilp<'a> { match self.inputs.contexts[context].kind { ContextKind::Query(_) => Scope::Queries, ContextKind::Ingestion(_) => Scope::Ingestion, - ContextKind::Shared(_) => Scope::SharedContexts, + ContextKind::Materialized(_) => Scope::SharedContexts, } } @@ -437,7 +437,7 @@ impl<'a> SelectionMilp<'a> { runs.push((context, candidate)); } } - if let ContextKind::Shared(own) = self.inputs.contexts[context].kind { + if let ContextKind::Materialized(own) = self.inputs.contexts[context].kind { if self.selected.contains_key(&own) { runs.push((context, own)); } @@ -573,7 +573,7 @@ fn choice_variables(milp: &mut SelectionMilp<'_>) { } } } - ContextKind::Shared(own) => { + ContextKind::Materialized(own) => { for (input, producers) in inputs.producer_candidates_per_input[own.0] .iter() .enumerate() @@ -622,9 +622,9 @@ fn choice_variables(milp: &mut SelectionMilp<'_>) { milp.selected.insert(shareable, selected); } for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(reader) = data.kind { + if let ContextKind::Materialized(reader) = data.kind { for &option in &data.options { - if inputs.kind[option.0] == RunKind::Shareable { + if inputs.kind[option.0] == RunKind::Materialized { milp.shared_readers.entry(option).or_default().push(reader); } } @@ -692,7 +692,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } } else { match kind { - RunKind::Shareable => { + RunKind::Materialized => { let selected = milp.selected[&option]; milp.builder .row("3 validity (boundary)", constraint!(variable <= selected)); @@ -716,7 +716,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } } } - if let ContextKind::Shared(own) = data.kind { + if let ContextKind::Materialized(own) = data.kind { let selected = milp.selected[&own]; for input in 0..inputs.producer_candidates_per_input[own.0].len() { let sum: Expression = inputs.producer_candidates_per_input[own.0][input] @@ -770,7 +770,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } Reader::Output(_) => reads += 1.0, Reader::OwnInput(_) => { - if let ContextKind::Shared(own) = data.kind { + if let ContextKind::Materialized(own) = data.kind { reads += milp.selected[&own]; } } @@ -792,7 +792,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { let mut reads_of: BTreeMap = BTreeMap::new(); for choices in &milp.choice { for (&option, &variable) in choices { - if inputs.kind[option.0] == RunKind::Shareable { + if inputs.kind[option.0] == RunKind::Materialized { *reads_of.entry(option).or_default() += variable; } } @@ -886,7 +886,7 @@ fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { // The node a read of `option` in `context` resolves to. let node_read = |context: usize, option: PhysicalCandidateId| -> Option { let run_context = match inputs.kind[option.0] { - RunKind::Shareable => milp.context_of(option)?, + RunKind::Materialized => milp.context_of(option)?, RunKind::Ingestion => milp.ingestion_context_read_by(context)?, RunKind::PerContext => return None, }; @@ -1047,7 +1047,7 @@ pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec) { let mut may_reach: BTreeMap> = BTreeMap::new(); for query in 0..milp.query_count() { for &option in &inputs.contexts[query].options { - if inputs.kind[option.0] == RunKind::Shareable { + if inputs.kind[option.0] == RunKind::Materialized { may_reach.entry(option).or_default().insert(query); } } @@ -1077,7 +1077,7 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { let queries = may_reach.get(&shareable).cloned().unwrap_or_default(); let context = inputs.context_of_shareable[&shareable]; for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable { + if inputs.kind[option.0] == RunKind::Materialized { may_reach .entry(option) .or_default() @@ -1184,14 +1184,14 @@ fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet) { (count, most) }; for (candidate, constraints) in constraints { - let shareable = inputs.kind[candidate.0] == RunKind::Shareable; + let shareable = inputs.kind[candidate.0] == RunKind::Materialized; // Its uses: (context, p) per context holding it as an option it // computes. let uses: Vec<(usize, Variable)> = (0..milp.choice.len()) @@ -1279,7 +1279,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { for &(context, used) in &uses { let count = match inputs.contexts[context].kind { ContextKind::Query(_) => Expression::from(1.0), - ContextKind::Shared(m) => reach_count(milp, &queries, m).0, + ContextKind::Materialized(m) => reach_count(milp, &queries, m).0, ContextKind::Ingestion(_) => { milp.note_inexact( "minimum readers of a candidate of an I_r (forbidden)", @@ -1333,7 +1333,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { for &(context, used) in &uses { match inputs.contexts[context].kind { ContextKind::Query(_) => {} - ContextKind::Shared(m) => { + ContextKind::Materialized(m) => { let (count, most) = reach_count(milp, &queries, m); let slack = (most - 1.0).max(0.0); milp.builder.row( @@ -1386,7 +1386,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { } } for &(context, used) in &uses { - if let ContextKind::Shared(m) = inputs.contexts[context].kind { + if let ContextKind::Materialized(m) = inputs.contexts[context].kind { if let (Some(&l), Some(&r)) = (milp.reach.get(&(left, m)), milp.reach.get(&(right, m))) { @@ -1422,7 +1422,7 @@ fn forbid_reach( .row("9 reader constraints", constraint!(used <= 0.0)); } ContextKind::Query(_) => {} - ContextKind::Shared(m) => { + ContextKind::Materialized(m) => { if let Some(&reach) = milp.reach.get(&(query, m)) { milp.builder .row("9 reader constraints", constraint!(reach + used <= 1.0)); @@ -1457,7 +1457,7 @@ fn context_work(milp: &mut SelectionMilp<'_>, context: usize) -> (Expression, f6 .copied() .filter(|&c| milp.computes(context, c)) .collect(); - if let ContextKind::Shared(own) = milp.inputs.contexts[context].kind { + if let ContextKind::Materialized(own) = milp.inputs.contexts[context].kind { runs.push(own); } // Per slot, only the slowest candidate can count (one per slot). @@ -1610,10 +1610,10 @@ fn accuracy_rows(milp: &mut SelectionMilp<'_>) { milp.choice[context].get(&candidate).map(|&p| p.into()) } ContextKind::Query(_) => None, - ContextKind::Shared(own) if own == candidate => { + ContextKind::Materialized(own) if own == candidate => { milp.reach.get(&(query, candidate)).map(|&a| a.into()) } - ContextKind::Shared(own) => match milp.reach.get(&(query, own)) { + ContextKind::Materialized(own) => match milp.reach.get(&(query, own)) { Some(&above) => { let p = milp.choice[context][&candidate]; Some( @@ -1644,7 +1644,7 @@ fn accuracy_rows(milp: &mut SelectionMilp<'_>) { }; if usable { places.insert(match inputs.contexts[context].kind { - ContextKind::Shared(own) if own == candidate => (context, None), + ContextKind::Materialized(own) if own == candidate => (context, None), _ => (context, Some(inputs.slot[candidate.0])), }); counted.push(( diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs index 41aa35e47..b7c6971d5 100644 --- a/crates/plan-selection/src/milp/model/tests.rs +++ b/crates/plan-selection/src/milp/model/tests.rs @@ -238,7 +238,7 @@ fn solve_and_check_by( let (plan, _) = plan_cost( plan_dag, &node_map(plan_dag), - &asap_cost::materialized_tumbling_window_chains(plan_dag), + &asap_cost::materialized_tumbling_window_chains(plan_dag).unwrap(), &recurrence, &ranges, &fixture.data, diff --git a/crates/plan-selection/src/milp/warm_start.rs b/crates/plan-selection/src/milp/warm_start.rs index cf2da1520..6a432cc85 100644 --- a/crates/plan-selection/src/milp/warm_start.rs +++ b/crates/plan-selection/src/milp/warm_start.rs @@ -364,7 +364,7 @@ impl Estimates { .map(|context| match context.kind { ContextKind::Query(query) => query_rate[query], ContextKind::Ingestion(_) => 1.0, - ContextKind::Shared(m) if context.at_query_time => { + ContextKind::Materialized(m) if context.at_query_time => { let readers = milp.queries_reaching(m); readers .iter() @@ -372,7 +372,7 @@ impl Estimates { .fold(f64::INFINITY, f64::min) .min(1e9) } - ContextKind::Shared(_) => 1.0, + ContextKind::Materialized(_) => 1.0, }) .collect(); let numbers = milp @@ -491,7 +491,7 @@ impl Derivation<'_, '_> { (context, false) } else { match kind { - RunKind::Shareable => match milp.context_of(candidate) { + RunKind::Materialized => match milp.context_of(candidate) { Some(own) => (own, plan.selected.contains(&candidate)), None => return f64::INFINITY, }, @@ -518,7 +518,8 @@ impl Derivation<'_, '_> { } Objective::Latency => numbers.latency_ms, }; - if self.amortized && kind == RunKind::Shareable && self.objective == Objective::Cost { + if self.amortized && kind == RunKind::Materialized && self.objective == Objective::Cost + { own /= self.estimates.readers[candidate.0].max(1) as f64; } let mut total = own; @@ -527,7 +528,7 @@ impl Derivation<'_, '_> { .iter() .filter(|producer| milp.choice[run_context].contains_key(producer)) .filter(|producer| { - kind == RunKind::Shareable + kind == RunKind::Materialized || inputs.slot[producer.0] != inputs.slot[candidate.0] }) .map(|&producer| self.estimate(plan, memo, run_context, producer)) @@ -635,7 +636,7 @@ impl Derivation<'_, '_> { return Ok(()); } match inputs.kind[candidate.0] { - RunKind::Shareable => { + RunKind::Materialized => { if plan.selected.contains(&candidate) { return Ok(()); } @@ -689,7 +690,7 @@ fn runs_reached( (context, false) } else { match inputs.kind[candidate.0] { - RunKind::Shareable => match milp.context_of(candidate) { + RunKind::Materialized => match milp.context_of(candidate) { Some(own) => (own, true), None => continue, }, diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index 4c26fc1f1..50ec26c93 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -305,6 +305,13 @@ pub enum ErrorSelectingPlan { roots: usize, entries: usize, }, + /// A candidate that cannot be costed as a whole; the error names the node + /// (one whose timing and lifetime cannot go together: a Stage 2 bug). + #[error("candidate {candidate}: {error}")] + InvalidCandidate { + candidate: String, + error: AnalyticalCostError, + }, #[error("candidate {candidate}, node {node:?}: {error}")] Cost { candidate: String, diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index 1da0aed75..789dfccd9 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -22,11 +22,14 @@ //! candidates' contexts apart), the timings, the warm start's cost, the //! objective, `plan_cost` and the dual bound (measurement only). -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::time::Instant; +use asap_cost::analytical::AnalyticalCostError; use asap_cost::{materialized_tumbling_window_chains, node_map, plan_cost, CostInputs}; +use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::{ASAPOp, Operator}; +use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, @@ -47,6 +50,32 @@ use crate::report::{ Selection, }; +/// [`ErrorSelectingPlan::Cost`] of `candidate`, from `plan_cost`'s error at +/// a node. +pub(crate) fn cost_error( + candidate: &str, +) -> impl Fn((PhysicalASAPNodeId, AnalyticalCostError)) -> ErrorSelectingPlan + '_ { + move |(node, error)| ErrorSelectingPlan::Cost { + candidate: candidate.to_owned(), + node, + error, + } +} + +/// `plan`'s [`materialized_tumbling_window_chains`], or +/// [`ErrorSelectingPlan::InvalidCandidate`] naming the node whose timing and +/// lifetime cannot go together. +pub(crate) fn materialized_chains_of( + plan: &PhysicalWorkloadPlan, +) -> Result, ErrorSelectingPlan> { + materialized_tumbling_window_chains(&plan.dag).map_err(|error| { + ErrorSelectingPlan::InvalidCandidate { + candidate: plan.id.clone(), + error, + } + }) +} + /// The largest relative difference between the objective and `plan_cost` /// that counts as equal. const COST_EQUALITY_TOLERANCE: f64 = 1e-6; @@ -128,18 +157,14 @@ pub fn select_plan( let (cost, _) = plan_cost( &plan.dag, &node_map(&plan.dag), - &materialized_tumbling_window_chains(&plan.dag), + &materialized_chains_of(plan)?, &recurrence, &ranges, data, cost_inputs, formula_scale, ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: plan.id.clone(), - node, - error, - })?; + .map_err(cost_error(&plan.id))?; let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) })? - solution.request_cost_per_second; diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs index c91c2917d..2344bb0b8 100644 --- a/crates/plan-selection/src/selector.rs +++ b/crates/plan-selection/src/selector.rs @@ -16,12 +16,10 @@ use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; use crate::report::{ AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, }; +use crate::select::{cost_error, materialized_chains_of}; use crate::SelectionModels; use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{ - materialized_tumbling_window_chains, node_map, plan_cost, query_reach, - tumbling_window_retention_ms, -}; +use asap_cost::{node_map, plan_cost, query_reach, tumbling_window_retention_ms}; type Ranking = PlanRanking; use asap_types::workload::bounds::with_confirmed_weights; @@ -181,7 +179,7 @@ impl<'a> PlanSelector<'a> { return Ok(CalibrationFit::default()); } let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_tumbling_window_chains(&candidate.dag); + let materialized_chains = materialized_chains_of(candidate)?; let (_, fit) = plan_cost( &candidate.dag, &nodes, @@ -192,11 +190,7 @@ impl<'a> PlanSelector<'a> { self.models.cost_inputs, 1.0, ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; + .map_err(cost_error(&candidate.id))?; Ok(fit) } @@ -231,7 +225,7 @@ impl<'a> PlanSelector<'a> { } }; let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_tumbling_window_chains(&candidate.dag); + let materialized_chains = materialized_chains_of(candidate)?; let settings = self.models.cost_inputs; let (cost, _) = plan_cost( &candidate.dag, @@ -243,11 +237,7 @@ impl<'a> PlanSelector<'a> { settings, self.formula_scale, ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; + .map_err(cost_error(&candidate.id))?; let violation = latency_violation(&cost, self.entries).or_else(|| match &settings.cost_model { CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits) diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 46fedf68d..bf82abbb1 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -95,11 +95,16 @@ impl PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, ) -> Result { + use ExecutionTiming::{IngestionTime, QueryTime}; match (timing, lifetime) { - (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + (IngestionTime, Lifetime::OneEvaluation(_)) => { Err(InvalidPhysicalProperties::OneEvaluationAtIngestion) } - _ => Ok(Self { timing, lifetime }), + (QueryTime | IngestionTime, Lifetime::None) + | (QueryTime, Lifetime::OneEvaluation(_)) + | (QueryTime | IngestionTime, Lifetime::AcrossEvaluations(_)) => { + Ok(Self { timing, lifetime }) + } } } @@ -123,10 +128,44 @@ impl PhysicalProperties { /// same timing, or a query-time reader of an ingestion-time output materialized /// until a query reads it. pub fn readable_at(self, reader: ExecutionTiming) -> bool { - reader == self.timing - || (reader == ExecutionTiming::QueryTime - && self.timing == ExecutionTiming::IngestionTime - && self.lifetime.is_across_evaluations()) + use ExecutionTiming::{IngestionTime, QueryTime}; + match (self.timing, self.lifetime, reader) { + // Runs inside the execution that reads it: only a reader at its + // own timing. + (QueryTime, Lifetime::None, QueryTime) => true, + (QueryTime, Lifetime::None, IngestionTime) => false, + (IngestionTime, Lifetime::None, IngestionTime) => true, + (IngestionTime, Lifetime::None, QueryTime) => false, + // Kept until a query reads it: ingestion work and queries both read it. + (IngestionTime, Lifetime::AcrossEvaluations(_), IngestionTime | QueryTime) => true, + // A query-time output: queries read it; ingestion work never does. + (QueryTime, Lifetime::OneEvaluation(_), QueryTime) => true, + (QueryTime, Lifetime::OneEvaluation(_), IngestionTime) => false, + (QueryTime, Lifetime::AcrossEvaluations(_), QueryTime) => true, + (QueryTime, Lifetime::AcrossEvaluations(_), IngestionTime) => false, + (IngestionTime, Lifetime::OneEvaluation(_), _) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + } + + /// The 8 valid states, as literal `(timing, lifetime)` pairs through + /// [`Self::new`], for table tests over every state. + #[cfg(any(test, feature = "test-support"))] + pub fn every_valid() -> [Self; 8] { + use ExecutionTiming::{IngestionTime, QueryTime}; + use Medium::{Disk, Memory}; + [ + (QueryTime, Lifetime::None), + (IngestionTime, Lifetime::None), + (QueryTime, Lifetime::OneEvaluation(Memory)), + (QueryTime, Lifetime::OneEvaluation(Disk)), + (QueryTime, Lifetime::AcrossEvaluations(Memory)), + (QueryTime, Lifetime::AcrossEvaluations(Disk)), + (IngestionTime, Lifetime::AcrossEvaluations(Memory)), + (IngestionTime, Lifetime::AcrossEvaluations(Disk)), + ] + .map(|(timing, lifetime)| Self::new(timing, lifetime).expect("a valid pair")) } } @@ -134,30 +173,35 @@ impl PhysicalProperties { mod tests { use super::*; + /// Of every timing x lifetime pair, exactly the 8 of `every_valid` are + /// valid, and the 2 at ingestion time for one evaluation are not. #[test] - fn one_evaluation_at_ingestion_is_invalid() { - assert_eq!( - PhysicalProperties::new( - ExecutionTiming::IngestionTime, - Lifetime::OneEvaluation(Medium::Memory) - ), - Err(InvalidPhysicalProperties::OneEvaluationAtIngestion) - ); - } - - #[test] - fn every_other_pair_is_valid() { - for timing in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] { + fn exactly_the_ingestion_time_one_evaluation_pairs_are_invalid() { + use ExecutionTiming::{IngestionTime, QueryTime}; + let mut valid = Vec::new(); + for timing in [QueryTime, IngestionTime] { for lifetime in [ Lifetime::None, Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), Lifetime::AcrossEvaluations(Medium::Disk), ] { - let valid = !(timing == ExecutionTiming::IngestionTime - && matches!(lifetime, Lifetime::OneEvaluation(_))); - assert_eq!(PhysicalProperties::new(timing, lifetime).is_ok(), valid); + match PhysicalProperties::new(timing, lifetime) { + Ok(properties) => valid.push(properties), + Err(error) => { + assert_eq!(error, InvalidPhysicalProperties::OneEvaluationAtIngestion); + assert!( + timing == IngestionTime + && matches!(lifetime, Lifetime::OneEvaluation(_)) + ); + } + } } } + let every_valid = PhysicalProperties::every_valid(); + assert_eq!(valid.len(), every_valid.len()); + assert!(valid.iter().all(|state| every_valid.contains(state))); } /// The named properties are valid pairs. @@ -177,15 +221,48 @@ mod tests { } } + /// `readable_at` for every valid state and both reader timings. #[test] - fn a_query_time_reader_reads_only_a_materialized_ingestion_output() { + fn readable_at_over_every_valid_state() { + use ExecutionTiming::{IngestionTime, QueryTime}; let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); - let query = ExecutionTiming::QueryTime; - let ingestion = ExecutionTiming::IngestionTime; - assert!(at(ingestion, Lifetime::AcrossEvaluations(Medium::Memory)).readable_at(query)); - assert!(!at(ingestion, Lifetime::None).readable_at(query)); - assert!(at(ingestion, Lifetime::None).readable_at(ingestion)); - assert!(!at(query, Lifetime::OneEvaluation(Medium::Memory)).readable_at(ingestion)); - assert!(at(query, Lifetime::AcrossEvaluations(Medium::Memory)).readable_at(query)); + // (state, readable by a query-time reader, readable by an ingestion-time reader) + let mut expected = Vec::new(); + for medium in [Medium::Memory, Medium::Disk] { + expected.extend([ + (at(QueryTime, Lifetime::OneEvaluation(medium)), true, false), + ( + at(QueryTime, Lifetime::AcrossEvaluations(medium)), + true, + false, + ), + ( + at(IngestionTime, Lifetime::AcrossEvaluations(medium)), + true, + true, + ), + ]); + } + expected.extend([ + (at(QueryTime, Lifetime::None), true, false), + (at(IngestionTime, Lifetime::None), false, true), + ]); + assert_eq!(expected.len(), PhysicalProperties::every_valid().len()); + for properties in PhysicalProperties::every_valid() { + let (_, at_query, at_ingestion) = expected + .iter() + .find(|(state, ..)| *state == properties) + .expect("every valid state has an expectation"); + assert_eq!( + properties.readable_at(QueryTime), + *at_query, + "{properties:?}" + ); + assert_eq!( + properties.readable_at(IngestionTime), + *at_ingestion, + "{properties:?}" + ); + } } } diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 0304f08b0..721835e9d 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -769,3 +769,60 @@ fn workload_export_matches_the_single_root_export() { let untimed = [QueryRoot::Operator(plan())]; assert!(compile_physical_asap_workload_with_node_ids(&untimed, &physical_properties).is_err()); } + +/// Export validation over every valid state of `plan()`'s summary build: +/// each materialized state, any medium, is assigned, exported and valid; not +/// materialized at query time is assigned and valid; not materialized at +/// ingestion time is valid for the scan below an ingestion-time build and, +/// for the build itself, rejected by rule (an ingestion-time state must be +/// materialized for a query to read it). +#[test] +fn export_validation_over_every_valid_state() { + use ExecutionTiming::{IngestionTime, QueryTime}; + for properties in PhysicalProperties::every_valid() { + let root = plan(); + let build = root.children()[0]; + let mut assignment = PhysicalPropertiesAssignment::default(); + let unmaterialized_at_ingestion = + (properties.timing(), properties.lifetime()) == (IngestionTime, Lifetime::None); + if !unmaterialized_at_ingestion { + assignment.set(build, properties); + } + let timed = apply_physical_properties(&root, &assignment).unwrap(); + let mut dag = compile_physical_asap_dag(&root, &timed).unwrap(); + let [scan, build, _finalize] = [0, 1, 2].map(|i| dag.nodes[i].physical_properties()); + match (properties.timing(), properties.lifetime()) { + (QueryTime, Lifetime::None) => { + assert_eq!(build, Ok(properties)); + dag.validate().unwrap(); + } + (IngestionTime, Lifetime::None) => { + dag.nodes[1].output_state.timing = IngestionTime; + assert_eq!( + dag.validate(), + Err( + PhysicalASAPDAGValidationError::UnmaterializedSummaryBuildNotAtQueryTime { + node: 1 + } + ) + ); + } + (IngestionTime, Lifetime::AcrossEvaluations(_)) => { + assert_eq!(build, Ok(properties)); + assert_eq!(scan, Ok(PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION)); + dag.validate().unwrap(); + } + (QueryTime, Lifetime::OneEvaluation(_)) => { + assert_eq!(build, Ok(properties)); + dag.validate().unwrap(); + } + (QueryTime, Lifetime::AcrossEvaluations(_)) => { + assert_eq!(build, Ok(properties)); + dag.validate().unwrap(); + } + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + } +} From d32bdc61fc004eac23c5a7050bab4b6674faa025 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:44:56 +0000 Subject: [PATCH 2/3] refactor: review fixes for every valid state; PhysicalRequirement and satisfies_physical Replaces readable_at with the interfaces #1200 and the concepts doc define: PhysicalRequirement { reader_timing, reads_from: Evaluation } (new rejects an ingestion-time reader of a later evaluation) and satisfies_physical(producer, requirement) for alpha^phys. is_materialized moves from Lifetime to PhysicalProperties, as the doc defines it; Lifetime::is_across_evaluations, now unused, is deleted. Review fixes: the realizer checks the whole selected (timing, lifetime) after the timing pass; ErrorSelectingPlan::InvalidCandidate and materialized_chains_of are deleted (materialized_tumbling_window_chains reports the node, as plan_cost does, through cost_error); the warm start uses materialized_run_rate; shareable/shared names in Stage 3 say materialized (kappa_m); is_query_time_materialized_tumbling_window_chain says what it tests; the labels say "query time, one evaluation" and "query time, materialized across evaluations"; remaining consumers match every valid state. Co-Authored-By: Claude Opus 5.5 --- crates/cost/src/execution_demand.rs | 89 ++++---- crates/cost/src/lib.rs | 2 +- crates/cost/src/usage.rs | 53 +++-- crates/cost/tests/ingestion_refresh_runs.rs | 2 +- crates/devtools/src/candidate_labels.rs | 14 +- .../fuzz-issue903-raw-retained.json | 2 +- crates/executor/src/physical_planner/mod.rs | 30 ++- .../tests/planner_layering_common/mod.rs | 2 +- .../tests/runtime_handoff.rs | 2 +- .../src/catalog/configurations.rs | 10 +- .../src/catalog/realize.rs | 23 +- crates/physical-asap-optimization/src/lib.rs | 22 +- crates/plan-selection/src/groups.rs | 128 ++++++----- crates/plan-selection/src/groups/tests.rs | 25 ++- crates/plan-selection/src/lib.rs | 2 +- crates/plan-selection/src/milp/cost_terms.rs | 76 +++++-- crates/plan-selection/src/milp/decode.rs | 28 +-- crates/plan-selection/src/milp/model.rs | 209 ++++++++++-------- crates/plan-selection/src/milp/model/tests.rs | 22 +- crates/plan-selection/src/milp/warm_start.rs | 66 +++--- crates/plan-selection/src/report.rs | 7 - crates/plan-selection/src/select.rs | 21 +- crates/plan-selection/src/selector.rs | 13 +- crates/planner/tests/runtime_install.rs | 2 +- crates/types/src/ir/materialized_view.rs | 35 ++- crates/types/src/ir/physical_export.rs | 14 +- crates/types/src/ir/properties/timing.rs | 18 +- crates/types/src/physical/mod.rs | 8 +- crates/types/src/physical/properties.rs | 209 ++++++++++++++---- crates/types/src/physical/workload_plan.rs | 2 +- crates/types/tests/tumbling_window_chain.rs | 2 +- .../develop_docs/extend-asap-aware-mapping.md | 2 +- 32 files changed, 730 insertions(+), 410 deletions(-) diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/execution_demand.rs index 4f8aa8d54..0b6700703 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/execution_demand.rs @@ -300,7 +300,17 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap = dag .nodes .iter() - .filter(|n| n.lifetime.is_materialized()) + .filter(|n| { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (n.output_state.timing, n.lifetime) { + (QueryTime | IngestionTime, Lifetime::None) => false, + (QueryTime, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_)) + | (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + }) .map(|n| n.id) .collect(); let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); @@ -342,8 +352,8 @@ pub enum MaterializedRunRate { /// At the combined evaluation rate of the queries reaching it, range /// steps included ([`evaluation_rate`]). PerEvaluation, - /// Once per request of the queries reaching it: a result kept across - /// evaluations builds only what is new (a chain's newest tumbling + /// Once per request of the queries reaching it: a result materialized + /// across evaluations builds only what is new (a chain's newest tumbling /// window), however many range steps a request evaluates. PerRequest, } @@ -374,7 +384,7 @@ pub fn materialized_run_rate(properties: PhysicalProperties) -> Option) -> QueryRecurrence { @@ -716,4 +691,40 @@ mod tests { Err(AnalyticalCostError::InvalidRecurrence) )); } + + /// How often each valid state is computed when materialized, and `None` for + /// the two not-materialized states. + #[test] + fn materialized_run_rate_over_every_valid_state() { + use asap_types::physical::Medium; + use ExecutionTiming::{IngestionTime, QueryTime}; + let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); + let mut expected = vec![ + (at(QueryTime, Lifetime::None), None), + (at(IngestionTime, Lifetime::None), None), + ]; + for medium in [Medium::Memory, Medium::Disk] { + expected.extend([ + ( + at(IngestionTime, Lifetime::AcrossEvaluations(medium)), + Some(MaterializedRunRate::PerSecondOfArrivingData), + ), + ( + at(QueryTime, Lifetime::OneEvaluation(medium)), + Some(MaterializedRunRate::PerEvaluation), + ), + ( + at(QueryTime, Lifetime::AcrossEvaluations(medium)), + Some(MaterializedRunRate::PerRequest), + ), + ]); + } + for properties in PhysicalProperties::every_valid() { + let (_, rate) = expected + .iter() + .find(|(state, _)| *state == properties) + .expect("every valid state has an expectation"); + assert_eq!(materialized_run_rate(properties), *rate, "{properties:?}"); + } + } } diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 708cb7ac9..ddf494add 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -41,7 +41,7 @@ pub use measured::{ pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, is_materialized_tumbling_window_chain, maintained_window_ms, + family_name, is_query_time_materialized_tumbling_window_chain, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, plan_cost, query_reach, retained_tumbling_windows, trailing_window, tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index a31d4fada..9c9b9c0e8 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -576,7 +576,19 @@ pub fn node_usage<'a>( let mut raw_scan = None; // A query-time scan's raw rows kept ([`ScanWindow::lookback_ms`]). let mut raw_kept_rows = None; - let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + let (timing, lifetime) = timing_and_lifetime(node)?; + // Whether the node computes at ingestion time, over arriving rows: the + // same for every lifetime. + let ingestion = match (timing, lifetime) { + (IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_)) => true, + ( + QueryTime, + Lifetime::None | Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), + ) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; let inputs: Vec = producers.iter().map(|p| p.output).collect(); let sources: BTreeSet<&str> = match &node.payload { Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), @@ -1140,10 +1152,9 @@ pub fn node_usage<'a>( node.payload, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ); - // A whole state kept across evaluations: it and the one being built. - let whole_state_kept = || state.checked_mul(2).ok_or(AnalyticalCostError::Overflow); - let (timing, lifetime) = timing_and_lifetime(node)?; - let (kept, medium) = match (timing, lifetime) { + // A whole state materialized across evaluations: it and the one being built. + let whole_state_materialized = || state.checked_mul(2).ok_or(AnalyticalCostError::Overflow); + let (materialized_bytes, medium) = match (timing, lifetime) { // Working memory while it runs, once per consumer that recomputes it, // and held through each such consumer's evaluation. (QueryTime, Lifetime::None) => { @@ -1160,7 +1171,7 @@ pub fn node_usage<'a>( peak_memory_bytes = estimate.peak_memory_bytes().saturating_mul(multiplier); (0, Medium::Memory) } - // A tumbling-window chain kept across evaluations holds the tumbling windows its + // A tumbling-window chain materialized across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window // width of the readers' range-evaluation span), all but the // newest between evaluations and all while one runs. @@ -1189,7 +1200,7 @@ pub fn node_usage<'a>( detail += &format!("; materializes {between} bytes across evaluations"); (0, medium) } - (QueryTime, Lifetime::AcrossEvaluations(medium)) => (whole_state_kept()?, medium), + (QueryTime, Lifetime::AcrossEvaluations(medium)) => (whole_state_materialized()?, medium), (QueryTime, Lifetime::OneEvaluation(medium)) => { match medium { Medium::Memory => { @@ -1214,7 +1225,9 @@ pub fn node_usage<'a>( medium, ) } - (IngestionTime, Lifetime::AcrossEvaluations(medium)) => (whole_state_kept()?, medium), + (IngestionTime, Lifetime::AcrossEvaluations(medium)) => { + (whole_state_materialized()?, medium) + } (IngestionTime, Lifetime::OneEvaluation(_)) => { unreachable!("rejected by PhysicalProperties::new") } @@ -1618,37 +1631,39 @@ pub type NodeMap<'a> = /// (lifetime `AcrossEvaluations` at query time, which Stage 2 offers only when the /// deployment keeps query-time results and every reader builds one new /// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the -/// older ones, which complete the readers' look-back, are kept from -/// earlier evaluations. Fails on a node whose timing and lifetime cannot -/// go together. +/// older ones, which complete the readers' look-back, are materialized from +/// earlier evaluations. Fails, naming the node, on a node whose timing and +/// lifetime cannot go together. pub fn materialized_tumbling_window_chains( dag: &PhysicalASAPDAG, -) -> Result, AnalyticalCostError> { +) -> Result, (PhysicalASAPNodeId, AnalyticalCostError)> { let mut chains = HashSet::new(); for node in &dag.nodes { - if is_materialized_tumbling_window_chain(node)? { + if is_query_time_materialized_tumbling_window_chain(node) + .map_err(|error| (node.id, error))? + { chains.insert(node.id); } } Ok(chains) } -/// Whether `node` is a tumbling-window chain kept across evaluations at -/// query time ([`materialized_tumbling_window_chains`]). Query time only: these are +/// Whether `node` is a tumbling-window chain materialized across evaluations +/// at query time ([`materialized_tumbling_window_chains`]). Query time only: these are /// the chains whose query-time scan reads just the newest tumbling window. -/// A chain kept across evaluations at ingestion time is built as data +/// A chain materialized across evaluations at ingestion time is built as data /// arrives, so no query-time scan reads below it; plan selection's /// retained-window options (`retained_pane_options`) and /// [`tumbling_window_retention_ms`] cover both timings, since a chain materialized /// at either timing retains windows. -pub fn is_materialized_tumbling_window_chain( +pub fn is_query_time_materialized_tumbling_window_chain( node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, ) -> Result { let chain = matches!( node.payload, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ); - let kept_at_query_time = match timing_and_lifetime(node)? { + let materialized_at_query_time = match timing_and_lifetime(node)? { (QueryTime, Lifetime::AcrossEvaluations(_)) => true, // Rebuilt whole by each evaluation that reads it: its scan reads // the whole window. @@ -1663,7 +1678,7 @@ pub fn is_materialized_tumbling_window_chain( unreachable!("rejected by PhysicalProperties::new") } }; - Ok(chain && kept_at_query_time) + Ok(chain && materialized_at_query_time) } /// `node`'s timing and lifetime, checked by [`PhysicalProperties::new`]: diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs index b5e33de37..75d76f92d 100644 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ b/crates/cost/tests/ingestion_refresh_runs.rs @@ -301,7 +301,7 @@ fn an_invalid_node_is_reported_not_skipped() { let chain = dag .nodes .iter_mut() - .find(|node| node.lifetime.is_materialized()) + .find(|node| node.lifetime != asap_types::physical::Lifetime::None) .unwrap(); chain.lifetime = asap_types::physical::Lifetime::OneEvaluation(asap_types::physical::Medium::Memory); diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index e18fd3ce6..196a926d4 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -141,7 +141,7 @@ fn duration_name(ms: u64) -> String { } /// `logical` followed by what the candidate materializes, e.g. -/// "… · ingestion time: Kll for Q1, 5 × 1m tumbling windows · query time, shared: +/// "… · ingestion time: Kll for Q1, 5 × 1m tumbling windows · query time, one evaluation: /// DDSketch"; /// just `logical` when nothing is. pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String { @@ -198,8 +198,10 @@ fn phase_label( (QueryTime, Lifetime::None) => None, (IngestionTime, Lifetime::None) => None, (IngestionTime, Lifetime::AcrossEvaluations(_)) => Some("ingestion time"), - (QueryTime, Lifetime::OneEvaluation(_)) => Some("query time, shared"), - (QueryTime, Lifetime::AcrossEvaluations(_)) => Some("query time, kept"), + (QueryTime, Lifetime::OneEvaluation(_)) => Some("query time, one evaluation"), + (QueryTime, Lifetime::AcrossEvaluations(_)) => { + Some("query time, materialized across evaluations") + } (IngestionTime, Lifetime::OneEvaluation(_)) => { unreachable!("rejected by PhysicalProperties::new") } @@ -517,7 +519,7 @@ mod tests { .iter() .filter(|node| { node.output_state.timing == ExecutionTiming::IngestionTime - && node.lifetime.is_materialized() + && node.lifetime != Lifetime::None && matches!( node.payload, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) @@ -576,11 +578,11 @@ mod tests { ), ( at(QueryTime, Lifetime::OneEvaluation(medium)), - Some("query time, shared"), + Some("query time, one evaluation"), ), ( at(QueryTime, Lifetime::AcrossEvaluations(medium)), - Some("query time, kept"), + Some("query time, materialized across evaluations"), ), ]); } diff --git a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json index f27b75631..67f877905 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json +++ b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json @@ -911,7 +911,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact · Q2 Kll · shared Scan (Q1, Q2) · query time, shared: TimeRange" + "display_name": "Q1 exact · Q2 Kll · shared Scan (Q1, Q2) · query time, one evaluation: TimeRange" }, "tumbling_window_retention_ms": {} } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 70d9f0428..3c84abee0 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -916,8 +916,21 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result true, + ( + QueryTime, + Lifetime::None | Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), + ) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + }; + if at_ingestion { let value = |schema: &SchemaRef| -> Result { let columns = schema .fields @@ -1730,7 +1743,18 @@ pub mod test_support { .nodes .iter() .filter(|node| { - nodes.contains(&(node.id as NodeId)) && !node.lifetime.is_across_evaluations() + use planner_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; + use planner_types::physical::Lifetime; + let materialized_across_evaluations = + match (node.output_state.timing, node.lifetime) { + (QueryTime | IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (QueryTime | IngestionTime, Lifetime::None) + | (QueryTime, Lifetime::OneEvaluation(_)) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; + nodes.contains(&(node.id as NodeId)) && !materialized_across_evaluations }) .map(|node| node.id as NodeId) .collect(); diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 35864cf2b..4fcad321c 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -783,7 +783,7 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { let materialized_nodes: BTreeSet<_> = dag .nodes .iter() - .filter(|n| n.lifetime.is_materialized()) + .filter(|n| n.lifetime != Lifetime::None) .map(|n| n.id) .collect(); // Each walk's start and, for a consumer refreshed at ingestion, its diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs index d3f1ba3d1..d6643b672 100644 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ b/crates/integration-tests/tests/runtime_handoff.rs @@ -64,7 +64,7 @@ async fn plan(workload: &PlanningWorkload, capabilities: &DeploymentCapabilities fn materialized_count(dag: &PhysicalASAPDAG) -> usize { dag.nodes .iter() - .filter(|n| n.lifetime.is_materialized()) + .filter(|n| n.lifetime != asap_types::physical::Lifetime::None) .count() } diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs index d78777e8f..d436675c6 100644 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ b/crates/physical-asap-optimization/src/catalog/configurations.rs @@ -13,7 +13,9 @@ use crate::summary_materialization_eligibility; use asap_types::ir::operator::{agg_has_exact_per_entity_kernel, Reduction}; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::{Lifetime, PhysicalProperties}; +use asap_types::physical::{ + satisfies_physical, Evaluation, Lifetime, PhysicalProperties, PhysicalRequirement, +}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use super::inventory::CandidateInventory; @@ -141,7 +143,7 @@ pub fn enumerate_candidate_configurations( // still handles them when given one): // - any `Disk` medium (#1217): the executor has no disk store, so a // `Disk` state given to it executes in memory (a view slot bound to - // the deployment's in-memory store, a one-evaluation result kept + // the deployment's in-memory store, a one-evaluation result materialized // within the query run), while cost prices it as disk; // - (IngestionTime, None) for a summary build: an unmaterialized build // runs at query time (`apply_physical_properties`); @@ -319,9 +321,11 @@ pub fn enumerate_candidate_configurations( .into_iter() .enumerate() { + let requirement = PhysicalRequirement::new(timing, Evaluation::Same) + .expect("a same-evaluation read is valid at either timing"); let mut compatible = Vec::new(); for &producer in configurations { - if candidates[producer.0].properties.readable_at(timing) { + if satisfies_physical(&candidates[producer.0].properties, &requirement) { compatible.push(producer); } } diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index 1ab1b6604..eaef1e232 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -68,10 +68,11 @@ pub fn realize_selected_candidates( }); // The option with the requested properties; option `None` (no // entry: not materialized, placed by the timing pass) for a - // requested lifetime `None`, at either timing. + // requested lifetime `None`, at either timing. The timing pass's + // placement is checked against the requested timing below. let chosen = options.iter().position(|option| match option { Some(entry) => *entry == requested, - None => !requested.lifetime().is_materialized(), + None => !requested.is_materialized(), }); if chosen.is_none() { let message = format!( @@ -100,7 +101,7 @@ pub fn realize_selected_candidates( let physical_outputs = ordered_output_references(&physical.roots); let identities = paired_node_identities(&rebuilt.outputs, &physical_outputs)?; for (node, properties) in assignments.iter().enumerate() { - if properties.lifetime().is_materialized() && !seen[node] { + if properties.is_materialized() && !seen[node] { // The decoder already established node identity. Stage 2 did // not offer this selected node (e.g. a lower shared non-summary), // so the assignment is infeasible rather than an adapter failure. @@ -122,9 +123,14 @@ pub fn realize_selected_candidates( let realized = realized_properties .get(&physical) .ok_or_else(|| format!("selected node {node} has no physical properties"))?; - if realized.lifetime() != assignments[node].lifetime() { + // The whole (timing, lifetime) Stage 3 selected and priced: an + // option `None` leaves the timing to the timing pass, which must + // place the node where Stage 3 assumed. + if realized != assignments[node] { return Err(format!( - "selected node {node} lifetime changed during physical reconstruction" + "selected node {node} has physical properties {realized:?} after physical \ + reconstruction, not the selected {:?}", + assignments[node] )); } Ok(physical) @@ -259,12 +265,7 @@ mod tests { let producer = producer_candidates .iter() .copied() - .find(|s| { - !physical.candidates[s.0] - .properties - .lifetime() - .is_materialized() - }) + .find(|s| !physical.candidates[s.0].properties.is_materialized()) .unwrap_or(producer_candidates[0]); let producer = select_from(physical, producer, selected); selected.edges.push(SelectedEdge { diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 87cb0d1bc..74a69b705 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -354,7 +354,7 @@ impl MaterializedOutputCandidate { /// which places it where its consumer runs), then materialized at /// ingestion time when eligible, `OneEvaluation` when two or more /// queries read it, and query-time `AcrossEvaluations` when it may be - /// kept across evaluations. Every materialized option is in memory: + /// materialized across evaluations. Every materialized option is in memory: /// Stage 2 offers no `Disk` state, since the executor runs a `Disk` /// state in memory (#1217). The other valid states it does not offer /// are listed at `enumerate_candidate_configurations`. @@ -938,8 +938,22 @@ fn down_closed( below: &[Vec], assignment: &[Option], ) -> bool { - let at_ingestion = |entry: &Option| { - entry.is_some_and(|p| p.timing() == ExecutionTiming::IngestionTime) + use asap_types::physical::Lifetime; + use ExecutionTiming::{IngestionTime, QueryTime}; + // Whether an option puts its node at ingestion time; option `None` + // leaves its node to the timing pass. + let at_ingestion = |entry: &Option| match entry + .map(|properties| (properties.timing(), properties.lifetime())) + { + None => false, + Some((IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_))) => true, + Some(( + QueryTime, + Lifetime::None | Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), + )) => false, + Some((IngestionTime, Lifetime::OneEvaluation(_))) => { + unreachable!("rejected by PhysicalProperties::new") + } }; assignment.iter().enumerate().all(|(c, m)| { !at_ingestion(m) @@ -1574,7 +1588,7 @@ mod tests { .nodes .iter() .map(|node| node.lifetime) - .filter(|lifetime| lifetime.is_materialized()) + .filter(|lifetime| *lifetime != Lifetime::None) .collect() } diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs index 26ab74444..af1a3b328 100644 --- a/crates/plan-selection/src/groups.rs +++ b/crates/plan-selection/src/groups.rs @@ -9,8 +9,8 @@ //! per execution, standing cost, held bytes and own latency, every number //! from [`asap_cost::node_usage`]. //! 3. **Owners** ([`OwnerRegion`]): one per query and one per possible -//! instance of each shareable candidate, each with its region of stateless -//! candidates, the links inside it and the shareable candidates it may read. +//! instance of each materialized candidate, each with its region of stateless +//! candidates, the links inside it and the materialized candidates it may read. //! 4. **Accuracy weights** ([`AccuracyWeight`]) per query and approximate //! candidate, and pairs excluded together, from the composition of the //! query's outputs. @@ -29,7 +29,7 @@ //! - for a query-time scan, the window each reader reads, through time //! shifts ([`window_read_through`], which follows //! [`PhysicalASAPDAG::scan_window`] for a plan with that one reader); -//! - for a pane chain kept beyond one evaluation, the panes it keeps, which +//! - for a pane chain materialized beyond one evaluation, the panes it retains, which //! grow with the range-evaluation span of the queries reaching it //! ([`asap_cost::tumbling_window_retention_ms`], [`asap_cost::retained_tumbling_windows`]). //! @@ -144,14 +144,14 @@ pub(crate) struct MilpInputs { pub(crate) slot: Vec, /// The execution contexts: one per query (`0..Q`, in workload order), /// then the ingestion pipelines `I_r` some context reads, ascending by - /// [`IngestionPipeline`], then one per shareable candidate some context + /// [`IngestionPipeline`], then one per materialized candidate some context /// may read, ascending by candidate. pub(crate) contexts: Vec, - /// Per shareable candidate with a context, its index in `contexts`. - pub(crate) context_of_shareable: BTreeMap, - /// Per ingestion-time shareable candidate whose context reads a + /// Per materialized candidate with a context, its index in `contexts`. + pub(crate) context_of_materialized: BTreeMap, + /// Per ingestion-time materialized candidate whose context reads a /// candidate of an `I_r`: the index of that `I_r` in `contexts`. - pub(crate) ingestion_context_of_shareable: BTreeMap, + pub(crate) ingestion_context_of_materialized: BTreeMap, /// Per query: `w(q, k)` for each approximate candidate `k` whose values /// reach its output. pub(crate) accuracy_weights: Vec>, @@ -196,7 +196,7 @@ pub(crate) struct CandidateData { /// at (through time shifts, [`window_read_through`]). Empty for any /// other candidate. pub(crate) windows_per_reader: Vec<(PhysicalCandidateId, Vec)>, - /// For a kept pane chain whose usage depends on the panes it retains + /// For a materialized pane chain whose usage depends on the panes it retains /// ([`UsageContext::PaneRetentionOfReachingQueries`]): its usage per /// retained pane count. Empty otherwise. pub(crate) usage_per_retained_panes: Vec<(u64, CandidateUsage)>, @@ -281,7 +281,7 @@ pub(crate) enum UsageContext { ProducerOutput, /// Which reader reads a query-time scan, so which window it scans. ScanWindowOfReader, - /// Which queries reach a kept pane chain, so how many panes it keeps. + /// Which queries reach a materialized pane chain, so how many panes it retains. PaneRetentionOfReachingQueries, /// A producer candidate on a cycle of candidate links was not costed yet, so /// its outputs were not among the options. @@ -443,7 +443,7 @@ impl CandidateLinks { // With one candidate per result slot per context, a candidate that // runs in a context cannot read its own slot there (only a self-loop // could): such links are dropped (a widened scan's `Project` lists - // its own memo group). A shareable candidate reads in its own + // its own memo group). A materialized candidate reads in its own // context, so it keeps them, but never reads itself. let slot = result_slots(physical); let producers_per_input = producers_per_input @@ -472,7 +472,7 @@ impl CandidateLinks { } /// Whether `candidate` is materialized: selected once, its own context. - fn is_shareable(&self, candidate: PhysicalCandidateId) -> bool { + fn is_materialized(&self, candidate: PhysicalCandidateId) -> bool { self.kind[candidate.0] == RunKind::Materialized } } @@ -567,8 +567,8 @@ pub(crate) fn prepare_milp_inputs_with( let usable = |class: PhysicalCandidateId| candidate_data[class.0].usage.is_ok(); let ExecutionContexts { contexts, - context_of_shareable, - ingestion_context_of_shareable, + context_of_materialized, + ingestion_context_of_materialized, } = execution_contexts( physical, &kind, @@ -597,8 +597,8 @@ pub(crate) fn prepare_milp_inputs_with( kind, slot, contexts, - context_of_shareable, - ingestion_context_of_shareable, + context_of_materialized, + ingestion_context_of_materialized, accuracy_weights, accuracy_excluded_pairs, } @@ -701,24 +701,24 @@ pub(crate) enum ContextKind { pub(crate) enum IngestionPipeline { /// The views of this refresh window. Window(RefreshWindow), - /// An ingestion-time shareable candidate whose window is unbounded or + /// An ingestion-time materialized candidate whose window is unbounded or /// empty: refreshed alone. Alone(PhysicalCandidateId), } -/// The [`IngestionPipeline`] of ingestion-time shareable candidate -/// `shareable`, from its refresh window ([`refresh_windows`]). +/// The [`IngestionPipeline`] of ingestion-time materialized candidate +/// `materialized`, from its refresh window ([`refresh_windows`]). pub(crate) fn ingestion_pipeline( - shareable: PhysicalCandidateId, + materialized: PhysicalCandidateId, refresh_windows: &HashMap>, ) -> IngestionPipeline { - match refresh_windows.get(&shareable) { + match refresh_windows.get(&materialized) { Some(Some(window)) => IngestionPipeline::Window(*window), - _ => IngestionPipeline::Alone(shareable), + _ => IngestionPipeline::Alone(materialized), } } -/// Each ingestion-time shareable candidate's refresh window, as +/// Each ingestion-time materialized candidate's refresh window, as /// `asap_cost::execution_demand::computations` computes it on the realized /// plan ([`PhysicalASAPDAG::refresh_windows`]): on the DAG of the candidates /// below it, each input read from its first producer candidate. That is @@ -731,7 +731,7 @@ fn refresh_windows( let views: Vec = (0..physical.candidates.len()) .map(PhysicalCandidateId) .filter(|&class| { - links.is_shareable(class) + links.is_materialized(class) && physical.candidates[class.0].properties.timing() == ExecutionTiming::IngestionTime }) @@ -807,12 +807,12 @@ pub(crate) fn producer_window_disagreements_of_all( #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ExecutionContext { pub(crate) kind: ContextKind, - /// Whether it runs at query time (a query, or a query-time shareable + /// Whether it runs at query time (a query, or a query-time materialized /// candidate's inputs): its `PerContext` candidates are computed in it. pub(crate) at_query_time: bool, /// The candidates that may give a result slot in it, ascending: each /// computed in it ([`ExecutionContext::computes`]) or read across its - /// boundary (a shareable candidate, or a candidate of an `I_r`). + /// boundary (a materialized candidate, or a candidate of an `I_r`). pub(crate) options: Vec, } @@ -830,12 +830,12 @@ impl ExecutionContext { impl MilpInputs { /// The context where context `context` reads a candidate of an `I_r`: - /// itself for an `I_r`, `I_{r(m)}` for ingestion-time shareable `m`'s + /// itself for an `I_r`, `I_{r(m)}` for ingestion-time materialized `m`'s /// context; `None` for any other context (it reads none). pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { match self.contexts[context].kind { ContextKind::Ingestion(_) => Some(context), - ContextKind::Materialized(m) => self.ingestion_context_of_shareable.get(&m).copied(), + ContextKind::Materialized(m) => self.ingestion_context_of_materialized.get(&m).copied(), ContextKind::Query(_) => None, } } @@ -844,14 +844,14 @@ impl MilpInputs { /// [`execution_contexts`]' result (see [`MilpInputs`] for each field). pub(crate) struct ExecutionContexts { pub(crate) contexts: Vec, - pub(crate) context_of_shareable: BTreeMap, - pub(crate) ingestion_context_of_shareable: BTreeMap, + pub(crate) context_of_materialized: BTreeMap, + pub(crate) ingestion_context_of_materialized: BTreeMap, } /// The execution contexts and their options (module documentation): from -/// each query's output candidates, each shareable candidate's producers +/// each query's output candidates, each materialized candidate's producers /// and, per ingestion pipeline `I_r`, every candidate of `I_r` an -/// ingestion-time shareable candidate of refresh window `r` +/// ingestion-time materialized candidate of refresh window `r` /// (`refresh_windows`) reads across its boundary, expanded through the /// inputs of the candidates each context computes. Candidates `usable` /// rejects (no usage) are left out. A query-time `PerContext` candidate @@ -908,10 +908,10 @@ pub(crate) fn execution_contexts( context.options = expand(&context, seeds); contexts.push(context); } - // Shareable contexts, discovered from the options, then each `I_r` from + // Materialized contexts, discovered from the options, then each `I_r` from // every candidate of `I_r` read across a boundary; repeated until // nothing new. - let mut context_of_shareable: BTreeMap = BTreeMap::new(); + let mut context_of_materialized: BTreeMap = BTreeMap::new(); let mut shared: Vec = Vec::new(); // Per `I_r`: its context and its seeds. let mut ingestion: BTreeMap< @@ -924,7 +924,7 @@ pub(crate) fn execution_contexts( let mut seeds: Vec<(IngestionPipeline, PhysicalCandidateId)> = Vec::new(); let unscanned = contexts.iter().chain(shared.iter()).skip(scanned); for context in unscanned.chain(ingestion.values().map(|(context, _)| context)) { - // Only an ingestion-time shareable candidate reads a candidate + // Only an ingestion-time materialized candidate reads a candidate // of an `I_r` (Stage 2's timing rule). let pipeline = match context.kind { ContextKind::Materialized(m) if !context.at_query_time => { @@ -935,7 +935,9 @@ pub(crate) fn execution_contexts( for &class in &context.options { match kind[class.0] { RunKind::Materialized => { - if !context_of_shareable.contains_key(&class) && !pending.contains(&class) { + if !context_of_materialized.contains_key(&class) + && !pending.contains(&class) + { pending.push(class); } } @@ -983,7 +985,7 @@ pub(crate) fn execution_contexts( .copied() .collect(), ); - context_of_shareable.insert(class, usize::MAX); + context_of_materialized.insert(class, usize::MAX); shared.push(context); } for pipeline in grown { @@ -997,14 +999,14 @@ pub(crate) fn execution_contexts( context_of_pipeline.insert(pipeline, contexts.len()); contexts.push(context); } - let mut ingestion_context_of_shareable = BTreeMap::new(); + let mut ingestion_context_of_materialized = BTreeMap::new(); for context in shared { if let ContextKind::Materialized(class) = context.kind { - context_of_shareable.insert(class, contexts.len()); + context_of_materialized.insert(class, contexts.len()); if !context.at_query_time { let pipeline = ingestion_pipeline(class, refresh_windows); if let Some(&at) = context_of_pipeline.get(&pipeline) { - ingestion_context_of_shareable.insert(class, at); + ingestion_context_of_materialized.insert(class, at); } } } @@ -1012,8 +1014,8 @@ pub(crate) fn execution_contexts( } ExecutionContexts { contexts, - context_of_shareable, - ingestion_context_of_shareable, + context_of_materialized, + ingestion_context_of_materialized, } } @@ -1088,7 +1090,7 @@ fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { /// The window a scan read through `path` (time shifts, then the reader that /// ends it) covers, as [`PhysicalASAPDAG::scan_window`] computes it for a /// plan where `path` is the scan's only reader: the shifts' offsets add up, -/// and a `TimeRange` reads its range, a kept pane chain one pane, any other +/// and a `TimeRange` reads its range, a pane chain materialized across evaluations at query time one pane, any other /// pane chain its whole window; any other reader reads `Unbounded`. fn window_read_through( path: &[&PhysicalASAPDAGNode], @@ -1104,7 +1106,7 @@ fn window_read_through( Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - }) if asap_cost::is_materialized_tumbling_window_chain(node)? => { + }) if asap_cost::is_query_time_materialized_tumbling_window_chain(node)? => { u128::from(*tumbling_window_width_ms) } Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { @@ -1127,7 +1129,7 @@ fn window_read_through( /// the unions of the window kinds its readers there read (one scan run read /// by several readers reads the union, `PhysicalASAPDAG::scan_window`): /// every union of the kinds. -/// A read as an output or as a shareable candidate's own input is +/// A read as an output or as a materialized candidate's own input is /// `Unbounded`. Distinct; `Unbounded` alone when nothing reads it. fn scan_window_options( scan: PhysicalCandidateId, @@ -1157,7 +1159,7 @@ fn scan_window_options( window_union(read.iter().copied()) }) }; - let shareable = kind[scan.0] == RunKind::Materialized; + let materialized = kind[scan.0] == RunKind::Materialized; let mut all_kinds: Vec = Vec::new(); for context in contexts { if !context.options.contains(&scan) { @@ -1176,7 +1178,7 @@ fn scan_window_options( } } } - // A shareable candidate reading the scan as its own input reads it + // A materialized candidate reading the scan as its own input reads it // at its own window; a read as an output reads it whole. if let ContextKind::Materialized(own) = context.kind { if producers_per_input[own.0] @@ -1196,7 +1198,7 @@ fn scan_window_options( if read_whole && !kinds.contains(&ScanWindow::Unbounded) { kinds.push(ScanWindow::Unbounded); } - if shareable { + if materialized { for window in kinds { if !all_kinds.contains(&window) { all_kinds.push(window); @@ -1206,7 +1208,7 @@ fn scan_window_options( } add_unions(&kinds, &mut windows); } - if shareable { + if materialized { add_unions(&all_kinds, &mut windows); } if windows.is_empty() { @@ -1306,9 +1308,10 @@ fn scan_windows_per_reader( Ok(per_reader) } -/// The panes kept pane chain `chain` may keep, one per query that can reach -/// it (its range-evaluation span), in `reaching` order; its pane count for -/// any other node. +/// The panes the pane chain `chain`, materialized across evaluations at +/// either timing, may retain, one per query that can reach it (its +/// range-evaluation span), in `reaching` order; its pane count for any +/// other node. fn retained_pane_options( chain: PhysicalCandidateId, nodes: &[PhysicalASAPDAGNode], @@ -1317,11 +1320,22 @@ fn retained_pane_options( entries: &[QueryWorkloadEntry], ) -> Vec { let node = &nodes[chain.0]; - let kept = matches!( + let chain_node = matches!( node.payload, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && node.lifetime.is_across_evaluations(); - if !kept { + ); + let materialized_across_evaluations = { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (node.output_state.timing, node.lifetime) { + (QueryTime | IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (QueryTime | IngestionTime, Lifetime::None) + | (QueryTime, Lifetime::OneEvaluation(_)) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + }; + if !(chain_node && materialized_across_evaluations) { return vec![asap_cost::tumbling_windows_of(node)]; } // The chain over the time shifts below it, which set its input offset. @@ -1953,7 +1967,7 @@ pub(crate) struct DominanceProfile<'a> { pub(crate) struct DominanceKey<'a> { pub(crate) group: MemoGroupId, pub(crate) properties: PhysicalProperties, - pub(crate) shareable: bool, + pub(crate) materialized: bool, pub(crate) producer_candidates_per_input: &'a [Vec], pub(crate) readers: Vec<(PhysicalCandidateId, usize)>, pub(crate) output_references: Vec, @@ -2004,7 +2018,7 @@ fn dominance_profiles<'a>( same: DominanceKey { group: physical.memo_group_of(copy), properties: candidate.properties, - shareable: links.is_shareable(copy), + materialized: links.is_materialized(copy), producer_candidates_per_input: producers_per_input, readers: std::mem::take(&mut readers[class]), output_references: std::mem::take(&mut output_refs_of[class]), diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs index 718ce32db..8ecd5be77 100644 --- a/crates/plan-selection/src/groups/tests.rs +++ b/crates/plan-selection/src/groups/tests.rs @@ -179,7 +179,7 @@ fn dominance_needs_every_condition() { let key = DominanceKey { group: MemoGroupId(3), properties: query_time_not_materialized(), - shareable: false, + materialized: false, producer_candidates_per_input: &producers, readers: vec![(PhysicalCandidateId(9), 0)], output_references: vec![0], @@ -225,9 +225,9 @@ fn dominance_needs_every_condition() { }, ), ( - "shareable", + "materialized", DominanceKey { - shareable: true, + materialized: true, ..key.clone() }, ), @@ -376,7 +376,7 @@ fn pruning_keeps_the_smallest_equal_class_and_incomparable_ones() { let key = DominanceKey { group: MemoGroupId(0), properties: query_time_not_materialized(), - shareable: false, + materialized: false, producer_candidates_per_input: &producers, readers: Vec::new(), output_references: vec![0], @@ -477,7 +477,7 @@ fn a_dominated_class_is_deleted_with_its_links() { // Execution contexts // --------------------------------------------------------------------------- -/// A leaf, a materialized (shareable) middle, and two queries each reading +/// A leaf, a materialized (materialized) middle, and two queries each reading /// the middle through its own stateless top; a third query reads only its /// own leaf. Candidate `i` runs operator node `i`. fn shared_middle() -> PhysicalASAPMemoCandidates { @@ -512,7 +512,7 @@ fn execution_contexts_on_a_hand_built_catalog() { let physical = shared_middle(); let inputs = prepared(&physical); let (leaf, middle, top_0, top_1, lone) = (id(0), id(1), id(2), id(3), id(4)); - assert!(CandidateLinks::new(&physical).is_shareable(middle)); + assert!(CandidateLinks::new(&physical).is_materialized(middle)); assert!(inputs.pruned_candidates.is_empty()); let contexts: Vec<(ContextKind, Vec)> = inputs .contexts @@ -528,7 +528,10 @@ fn execution_contexts_on_a_hand_built_catalog() { (ContextKind::Materialized(middle), vec![leaf]), ] ); - assert_eq!(inputs.context_of_shareable, BTreeMap::from([(middle, 3)])); + assert_eq!( + inputs.context_of_materialized, + BTreeMap::from([(middle, 3)]) + ); assert!(inputs.contexts[3].computes(RunKind::PerContext)); assert!(!inputs.contexts[3].computes(RunKind::Materialized)); assert_eq!( @@ -641,14 +644,14 @@ fn two_queries_share_a_pane_chain_through_the_real_pipeline() { matches!( physical.physical_node(*candidate).asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && links.is_shareable(*candidate) + ) && links.is_materialized(*candidate) && inputs.contexts[0].options.contains(candidate) && inputs.contexts[1].options.contains(candidate) }) .collect(); assert!(!shared_chains.is_empty()); for &chain in &shared_chains { - let context = inputs.context_of_shareable[&chain]; + let context = inputs.context_of_materialized[&chain]; assert_eq!( inputs.contexts[context].kind, ContextKind::Materialized(chain) @@ -1078,10 +1081,10 @@ fn execution_contexts_have_one_ingestion_pipeline_per_refresh_window() { ] ); assert_eq!( - contexts.ingestion_context_of_shareable, + contexts.ingestion_context_of_materialized, BTreeMap::from([(id(2), 5), (id(3), 5), (id(5), 7), (id(8), 6), (id(9), 8)]) ); - let shared = contexts.context_of_shareable[&id(8)]; + let shared = contexts.context_of_materialized[&id(8)]; assert_eq!(contexts.contexts[shared].options, vec![id(7)]); } diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 18e72d90b..5ba637e8f 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -4840,7 +4840,7 @@ mod tests { c.dag.nodes.iter().all(|n| match n.payload { Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { n.output_state.timing == ExecutionTiming::IngestionTime - && n.lifetime.is_materialized() + && n.lifetime != Lifetime::None } _ => n.lifetime == Lifetime::None, }) diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs index 842dec882..c8e93883a 100644 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ b/crates/plan-selection/src/milp/cost_terms.rs @@ -8,7 +8,7 @@ //! coefficients and the auxiliary rows that linearize them. //! //! A **run** is a candidate computed in an execution context (`p[ctx, c]`), -//! or a shareable candidate's own run (`x[m]`, keyed by its own context). +//! or a materialized candidate's own run (`x[m]`, keyed by its own context). //! A run's usage is one or more [`RunTerm`]s, each an indicator and the //! numbers that count when it is 1: //! - one term on the run's own indicator when its usage is plan @@ -24,8 +24,8 @@ //! union is costed by `asap-cost` itself. //! //! **Executions per second** of a run: `rate(q)` in query context `q`; 1 in -//! each `I_r` and for an ingestion-time shareable candidate; for a query-time -//! shareable candidate `m` and the runs in its context, `Σ rate · z[m, G, +//! each `I_r` and for an ingestion-time materialized candidate; for a query-time +//! materialized candidate `m` and the runs in its context, `Σ rate · z[m, G, //! ℓ]` over its schedule groups and their levels, `z ≥ a[q, m]` for each //! reader at that level or faster (a group's rate is its fastest reader's). //! A context's runs other than `m`'s plain own term are summed once, `E[m] @@ -670,7 +670,7 @@ fn window_terms( terms } -/// For a shareable kept pane chain with usage depending on the panes it +/// For a materialized kept pane chain with usage depending on the panes it /// retains: its base term at the fewest panes, plus one term per larger /// retention, `π[m, ℓ] ≥` the reach of each query needing at least `ℓ`. fn pane_retention_terms( @@ -707,13 +707,13 @@ fn pane_retention_terms( indicators.push(direct.into()); } for &reader in milp - .shared_readers + .materialized_readers .get(&chain) .cloned() .unwrap_or_default() .iter() { - let reader_context = milp.inputs.context_of_shareable[&reader]; + let reader_context = milp.inputs.context_of_materialized[&reader]; if !milp.runs_at_query_time(reader_context) { continue; } @@ -770,7 +770,7 @@ fn pane_retention_terms( } } -/// Each ingestion-time shareable candidate's compaction term for its +/// Each ingestion-time materialized candidate's compaction term for its /// readers' latency, ms, where `plan_cost` charges one: `(k − 1) · states · /// merge time`, `k` the cores its ingestion work takes. With at most one /// second of ingestion work per second below it, `k = 1` and the term is 0; @@ -778,14 +778,14 @@ fn pane_retention_terms( pub(crate) fn compaction_ms(milp: &mut SelectionMilp<'_>) -> HashMap { let inputs = milp.inputs; let mut compaction = HashMap::new(); - for &shareable in inputs.context_of_shareable.keys() { - if milp.physical.candidates[shareable.0].properties.timing() + for &materialized in inputs.context_of_materialized.keys() { + if milp.physical.candidates[materialized.0].properties.timing() != ExecutionTiming::IngestionTime { continue; } let mut seen = BTreeSet::new(); - let mut stack = vec![shareable]; + let mut stack = vec![materialized]; let mut most_work_ms = 0.0; while let Some(class) = stack.pop() { if !seen.insert(class) { @@ -804,7 +804,7 @@ pub(crate) fn compaction_ms(milp: &mut SelectionMilp<'_>) -> HashMap 1_000.0 { milp.note_inexact("compaction with more than one ingest worker (not encoded)"); } - compaction.insert(shareable, 0.0); + compaction.insert(materialized, 0.0); } compaction } @@ -816,9 +816,9 @@ type GroupLevels = (Vec<(usize, f64)>, Vec); #[derive(Debug, Clone)] enum ContextRate { /// A fixed rate (a query's, 1 at ingestion time, or a query-time - /// shareable candidate whose possible readers run it at one rate). + /// materialized candidate whose possible readers run it at one rate). Fixed(f64), - /// `Σ rate · z` over a query-time shareable candidate's schedule-group + /// `Σ rate · z` over a query-time materialized candidate's schedule-group /// levels. Components(Vec<(Variable, f64)>), } @@ -839,7 +839,7 @@ fn context_rates( .collect::>() .map_err(cost)?; let mut rates = Vec::with_capacity(milp.inputs.contexts.len()); - milp.builder.scope = Scope::SharedContexts; + milp.builder.scope = Scope::MaterializedContexts; for context in 0..milp.inputs.contexts.len() { let rate = match milp.inputs.contexts[context].kind { ContextKind::Query(query) => ContextRate::Fixed(query_rate[query]), @@ -988,12 +988,12 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec } } }; - // Per query-time shareable context: the execution cost of its runs + // Per query-time materialized context: the execution cost of its runs // other than `m`'s plain own term, and per slot (`None`: `m`'s own run) // the most it can be. let mut context_execution: BTreeMap, f64>)> = BTreeMap::new(); - // `H[m]` per query-time shareable context. + // `H[m]` per query-time materialized context. let mut held_time: HashMap = HashMap::new(); let runs = milp.runs(); for (context, candidate) in runs { @@ -1005,6 +1005,25 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec candidate_data.properties.lifetime(), ); let own = milp.inputs.contexts[context].kind == ContextKind::Materialized(candidate); + // Whether a query evaluation holds the run's bytes (only at query + // time), and whether they are held exactly as priced: in memory for + // one evaluation. + let (held_at_query_time, held_in_memory_for_one_evaluation) = { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (timing, lifetime) { + (QueryTime, Lifetime::OneEvaluation(Medium::Memory)) => (true, true), + ( + QueryTime, + Lifetime::None + | Lifetime::OneEvaluation(Medium::Disk) + | Lifetime::AcrossEvaluations(_), + ) => (true, false), + (IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_)) => (false, false), + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } + }; // The most this run adds to its context's execution cost: all its // terms at 1. let mut run_execution = 0.0; @@ -1029,7 +1048,7 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec } } // Held memory. - if numbers.held_bytes <= 0.0 || timing != ExecutionTiming::QueryTime { + if numbers.held_bytes <= 0.0 || !held_at_query_time { continue; } // Per byte held per latency unit. @@ -1049,8 +1068,8 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec charge(&held.into(), per_byte_ms * query_rate[query]); } ContextKind::Materialized(m) => { - if own && lifetime != Lifetime::OneEvaluation(Medium::Memory) { - milp.note_inexact("held bytes of a shareable candidate not in memory"); + if own && !held_in_memory_for_one_evaluation { + milp.note_inexact("held bytes of a materialized candidate not in memory"); } let (longest, most_held) = *held_time.entry(context).or_insert_with(|| { let longest = milp @@ -1113,7 +1132,7 @@ pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), Selec } // A context's execution at its varying rate: `V ≥ E − Ē (x[m] − z)` per // component `z`, `Ē` the most `E` can be: per slot, its costliest run. - milp.builder.scope = Scope::SharedContexts; + milp.builder.scope = Scope::MaterializedContexts; for (context, (sum, most_per_slot)) in context_execution { let most: f64 = if super::model::tests_loose_big_m() { // Before: every term's cost. @@ -1181,8 +1200,21 @@ fn raw_retention( else { continue; }; - if milp.physical.candidates[candidate.0].properties.timing() != ExecutionTiming::QueryTime { - continue; + let properties = milp.physical.candidates[candidate.0].properties; + { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (properties.timing(), properties.lifetime()) { + // A query-time scan retains its raw data, whatever its lifetime. + ( + QueryTime, + Lifetime::None | Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), + ) => {} + // An ingestion-time scan reads arriving rows. + (IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_)) => continue, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } } let source = source_name(source).to_string(); for term in milp.run_terms(context, candidate) { diff --git a/crates/plan-selection/src/milp/decode.rs b/crates/plan-selection/src/milp/decode.rs index f764d66e7..3d1f78724 100644 --- a/crates/plan-selection/src/milp/decode.rs +++ b/crates/plan-selection/src/milp/decode.rs @@ -2,15 +2,15 @@ //! solution as a [`DagOfSelectedCandidates`], one node per candidate run. //! //! - A candidate computed in a context (`p[ctx, c] = 1`, a not-materialized -//! query-time candidate in a query's or a shareable candidate's context, +//! query-time candidate in a query's or a materialized candidate's context, //! or a not-materialized ingestion-time candidate in an `I_r`) is one node per //! context: selected in two contexts, it is two nodes; read by two //! readers in one context, it is one. -//! - A selected shareable candidate (`x[m] = 1`) is one node, whoever reads +//! - A selected materialized candidate (`x[m] = 1`) is one node, whoever reads //! it. //! - In each context, input `n` of a computed node (or of `m`'s own node in //! `m`'s context) reads the context's selected candidate among its -//! producers: the node computed there, the shareable candidate's node, or +//! producers: the node computed there, the materialized candidate's node, or //! the candidate's node in `I_r(m)` (read in ingestion-time `m`'s context). //! - Output reference `r` is the selected output candidate in `r`'s query //! context. @@ -33,12 +33,12 @@ use crate::groups::{MilpInputs, ResultSlot, RunKind}; enum Run { /// Candidate `.1` computed in context `.0`. Computed(usize, PhysicalCandidateId), - /// Shareable candidate `.0`. - Shared(PhysicalCandidateId), + /// Materialized candidate `.0`. + Materialized(PhysicalCandidateId), } /// The selected candidates of a solution: per context, `p[ctx, c] = 1`, and -/// the shareable candidates with `x[m] = 1`. +/// the materialized candidates with `x[m] = 1`. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub(crate) struct ChosenCandidates { pub(crate) chosen: Vec>, @@ -69,7 +69,7 @@ pub(crate) fn decode( RunKind::Materialized => chosen .selected .contains(&producer) - .then_some(Run::Shared(producer)), + .then_some(Run::Materialized(producer)), RunKind::Ingestion => inputs .ingestion_context_read_by(context) .filter(|&ingestion| chosen.chosen[ingestion].contains(&producer)) @@ -81,11 +81,13 @@ pub(crate) fn decode( let inputs_of = |run: Run| -> Result, String> { let (context, candidate) = match run { Run::Computed(context, candidate) => (context, candidate), - Run::Shared(candidate) => ( + Run::Materialized(candidate) => ( *inputs - .context_of_shareable + .context_of_materialized .get(&candidate) - .ok_or_else(|| format!("shareable candidate {} has no context", candidate.0))?, + .ok_or_else(|| { + format!("materialized candidate {} has no context", candidate.0) + })?, candidate, ), }; @@ -96,12 +98,12 @@ pub(crate) fn decode( { let slot_read = match run { Run::Computed(..) => chosen.slot_reads.get(&(context, candidate, input)), - Run::Shared(_) => None, + Run::Materialized(_) => None, }; let producer = producers .iter() .filter(|producer| { - matches!(run, Run::Shared(_)) + matches!(run, Run::Materialized(_)) || inputs.slot[producer.0] != inputs.slot[candidate.0] }) .filter(|producer| slot_read.is_none_or(|&slot| inputs.slot[producer.0] == slot)) @@ -137,7 +139,7 @@ pub(crate) fn decode( } node_of.insert(run, SelectedNodeId(candidate_of_node.len())); candidate_of_node.push(match run { - Run::Computed(_, candidate) | Run::Shared(candidate) => candidate, + Run::Computed(_, candidate) | Run::Materialized(candidate) => candidate, }); let reads = inputs_of(run)?; stack.extend(reads.iter().map(|&(_, producer)| producer)); diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 479bd939a..292453074 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -5,11 +5,11 @@ //! //! | Family | Meaning | //! |---|---| -//! | `p[ctx, c]` | in context `ctx`, candidate `c` gives its result slot (binary): computed there, or read across the boundary (a shareable candidate, or a candidate of an ingestion pipeline `I_r`) | -//! | `x[m]` | shareable candidate `m` is selected (binary) | -//! | `a[q, m]` | query `q` depends on shareable `m` (continuous, exact at integral binaries) | +//! | `p[ctx, c]` | in context `ctx`, candidate `c` gives its result slot (binary): computed there, or read across the boundary (a materialized candidate, or a candidate of an ingestion pipeline `I_r`) | +//! | `x[m]` | materialized candidate `m` is selected (binary) | +//! | `a[q, m]` | query `q` depends on materialized `m` (continuous, exact at integral binaries) | //! | `y[q, m, c]` | `AND(a[q, m], p[m, c])`, for reach from above and accuracy (continuous) | -//! | `L[ctx]`, `T[q, m]`, `K[q]` | latency of a query or of a query-time shareable candidate's run and context; `m`'s latency in `q`'s; compaction (continuous) | +//! | `L[ctx]`, `T[q, m]`, `K[q]` | latency of a query or of a query-time materialized candidate's run and context; `m`'s latency in `q`'s; compaction (continuous) | //! | `N[q]`, `A[q, parts]` | approximate values `q` depends on; whether it depends on one sized for `parts` shares (continuous) | //! | `g[m, c]` | highest shared materialization: `c` is read in context `m`, which at most one query reaches (binary) | //! | `rank` | order ranks on a context's cyclic links (continuous) | @@ -96,13 +96,13 @@ impl Default for SelectionMilpOptions { } /// Which part of the model a variable or row belongs to, for the size -/// report: the query contexts, the `I_r`, the shareable candidates' contexts +/// report: the query contexts, the `I_r`, the materialized candidates' contexts /// (with their `x`, reach and schedule terms), or the rest. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] pub(crate) enum Scope { Queries, Ingestion, - SharedContexts, + MaterializedContexts, Global, } @@ -218,7 +218,7 @@ impl ModelBuilder { /// One way a candidate's input is read in a context: `(reader, input)` /// with the reader computed in that context, the query's output reference, -/// or the shareable candidate's own input. +/// or the materialized candidate's own input. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub(crate) enum Reader { Candidate(PhysicalCandidateId, usize), @@ -243,13 +243,13 @@ pub(crate) struct SelectionMilp<'a> { pub(crate) choice: Vec>, /// Per context, per option, the reads of it in that context. pub(crate) readers: Vec>>, - /// `x[m]` per shareable candidate with a context. + /// `x[m]` per materialized candidate with a context. pub(crate) selected: BTreeMap, - /// `a[q, m]`, for each query that may depend on shareable `m`. + /// `a[q, m]`, for each query that may depend on materialized `m`. pub(crate) reach: HashMap<(usize, PhysicalCandidateId), Variable>, - /// Per shareable candidate, the shareable candidates whose contexts may + /// Per materialized candidate, the materialized candidates whose contexts may /// read it. - pub(crate) shared_readers: BTreeMap>, + pub(crate) materialized_readers: BTreeMap>, /// `L[q]`. pub(crate) latency: Vec, /// An upper bound of each `L[q]` (its bound, or the most it can be), in @@ -267,7 +267,7 @@ pub(crate) struct SelectionMilp<'a> { pub(crate) request_cost_per_second: f64, /// Terms encoded inexactly, by reason. pub(crate) inexact: BTreeMap<&'static str, usize>, - /// The terms of each run `(context, candidate)`, cached; a shareable + /// The terms of each run `(context, candidate)`, cached; a materialized /// candidate's own run is keyed by its own context. pub(crate) run_terms_cache: HashMap<(usize, PhysicalCandidateId), Vec>, /// Runs whose terms are being built (cycles of links). @@ -313,7 +313,7 @@ pub(crate) struct SelectionMilpSolution { pub(crate) cost_lower_bound_per_second: f64, /// Per context, the options with `p[ctx, c] = 1`. pub(crate) chosen: Vec>, - /// The shareable candidates with `x[m] = 1`. + /// The materialized candidates with `x[m] = 1`. pub(crate) selected: BTreeSet, /// Per multi-slot input `(context, reader, input)` of a used reader, the /// slot it reads. @@ -341,9 +341,12 @@ impl<'a> SelectionMilp<'a> { self.inputs.ingestion_context_read_by(context) } - /// The context of shareable candidate `m`. - pub(crate) fn context_of(&self, shareable: PhysicalCandidateId) -> Option { - self.inputs.context_of_shareable.get(&shareable).copied() + /// The context of materialized candidate `m`. + pub(crate) fn context_of(&self, materialized: PhysicalCandidateId) -> Option { + self.inputs + .context_of_materialized + .get(&materialized) + .copied() } pub(crate) fn kind(&self, candidate: PhysicalCandidateId) -> RunKind { @@ -360,7 +363,7 @@ impl<'a> SelectionMilp<'a> { } /// The indicator of run `(context, candidate)`: `p[ctx, c]` for a - /// candidate computed there, `x[m]` for a shareable candidate's own run + /// candidate computed there, `x[m]` for a materialized candidate's own run /// in its own context. pub(crate) fn run_indicator( &self, @@ -377,7 +380,7 @@ impl<'a> SelectionMilp<'a> { } /// The context where run `(context, candidate)` reads its inputs: its - /// own (a shareable candidate's own run reads in its context too). + /// own (a materialized candidate's own run reads in its context too). pub(crate) fn runs_at_query_time(&self, context: usize) -> bool { self.inputs.contexts[context].at_query_time } @@ -404,7 +407,7 @@ impl<'a> SelectionMilp<'a> { match self.inputs.contexts[context].kind { ContextKind::Query(_) => Scope::Queries, ContextKind::Ingestion(_) => Scope::Ingestion, - ContextKind::Materialized(_) => Scope::SharedContexts, + ContextKind::Materialized(_) => Scope::MaterializedContexts, } } @@ -428,7 +431,7 @@ impl<'a> SelectionMilp<'a> { } /// Every run of the model: `(context, candidate)` for each candidate - /// computed in a context, and each shareable candidate's own run. + /// computed in a context, and each materialized candidate's own run. pub(crate) fn runs(&self) -> Vec<(usize, PhysicalCandidateId)> { let mut runs = Vec::new(); for (context, options) in self.choice.iter().enumerate() { @@ -446,10 +449,10 @@ impl<'a> SelectionMilp<'a> { runs } - /// The query contexts that may depend on shareable `m`, ascending. - pub(crate) fn queries_reaching(&self, shareable: PhysicalCandidateId) -> Vec { + /// The query contexts that may depend on materialized `m`, ascending. + pub(crate) fn queries_reaching(&self, materialized: PhysicalCandidateId) -> Vec { let mut queries: Vec = (0..self.query_count()) - .filter(|&query| self.reach.contains_key(&(query, shareable))) + .filter(|&query| self.reach.contains_key(&(query, materialized))) .collect(); queries.sort_unstable(); queries @@ -495,7 +498,7 @@ pub(crate) fn build_selection_milp<'a>( readers: Vec::new(), selected: BTreeMap::new(), reach: HashMap::new(), - shared_readers: BTreeMap::new(), + materialized_readers: BTreeMap::new(), latency: Vec::new(), latency_max: Vec::new(), ms_per_latency_unit: 1.0, @@ -616,16 +619,19 @@ fn choice_variables(milp: &mut SelectionMilp<'_>) { milp.choice.push(choices); milp.readers.push(readers); } - milp.builder.scope = Scope::SharedContexts; - for &shareable in inputs.context_of_shareable.keys() { - let selected = milp.builder.binary("x (shareable candidates)"); - milp.selected.insert(shareable, selected); + milp.builder.scope = Scope::MaterializedContexts; + for &materialized in inputs.context_of_materialized.keys() { + let selected = milp.builder.binary("x (materialized candidates)"); + milp.selected.insert(materialized, selected); } for (context, data) in inputs.contexts.iter().enumerate() { if let ContextKind::Materialized(reader) = data.kind { for &option in &data.options { if inputs.kind[option.0] == RunKind::Materialized { - milp.shared_readers.entry(option).or_default().push(reader); + milp.materialized_readers + .entry(option) + .or_default() + .push(reader); } } let _ = context; @@ -638,14 +644,14 @@ fn choice_variables(milp: &mut SelectionMilp<'_>) { fn structural_rows(milp: &mut SelectionMilp<'_>) { let inputs = milp.inputs; let output_refs = output_references(&milp.physical.query_roots); - // Per `I_r`, the contexts of the ingestion-time shareable candidates of + // Per `I_r`, the contexts of the ingestion-time materialized candidates of // refresh window `r`. let mut readers_of_ingestion: BTreeMap> = BTreeMap::new(); - for (m, &at) in &inputs.ingestion_context_of_shareable { + for (m, &at) in &inputs.ingestion_context_of_materialized { readers_of_ingestion .entry(at) .or_default() - .push(inputs.context_of_shareable[m]); + .push(inputs.context_of_materialized[m]); } // 1. Coverage. milp.builder.scope = Scope::Queries; @@ -777,7 +783,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } } // A candidate of `I_r` is also read by the contexts of the - // ingestion-time shareable candidates of refresh window `r`. + // ingestion-time materialized candidates of refresh window `r`. for &other in readers_of_ingestion.get(&context).into_iter().flatten() { if let Some(&read) = milp.choice[other].get(&option) { reads += read; @@ -788,7 +794,7 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } } // 5. Sharing. - milp.builder.scope = Scope::SharedContexts; + milp.builder.scope = Scope::MaterializedContexts; let mut reads_of: BTreeMap = BTreeMap::new(); for choices in &milp.choice { for (&option, &variable) in choices { @@ -798,8 +804,8 @@ fn structural_rows(milp: &mut SelectionMilp<'_>) { } } let selected = milp.selected.clone(); - for (shareable, selected) in selected { - let reads = reads_of.remove(&shareable).unwrap_or_default(); + for (materialized, selected) in selected { + let reads = reads_of.remove(&materialized).unwrap_or_default(); milp.builder .row("5 sharing", constraint!(selected <= reads)); } @@ -858,18 +864,18 @@ fn acyclic_rows(milp: &mut SelectionMilp<'_>) { milp.builder.scope = Scope::Global; } -/// Order ranks across contexts: on the graph whose nodes are the shareable +/// Order ranks across contexts: on the graph whose nodes are the materialized /// candidates' own runs and the runs of each `I_r`, with an edge from a run -/// to each shareable candidate or candidate of an `I_r` its context may +/// to each materialized candidate or candidate of an `I_r` its context may /// read (for a run of `I_r`, its producers there). Rows 3–5 alone admit a /// cycle of reads across contexts that justifies itself with no output -/// above it (two shareable candidates of one memo group, each listed among +/// above it (two materialized candidates of one memo group, each listed among /// the other's producers); only edges inside a cyclic component get a row. fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { let inputs = milp.inputs; // Nodes: (context of the run, candidate). let mut nodes: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (&m, &context) in &inputs.context_of_shareable { + for (&m, &context) in &inputs.context_of_materialized { nodes.push((context, m)); } for (context, data) in inputs.contexts.iter().enumerate() { @@ -919,7 +925,7 @@ fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { } let plain: Vec<(usize, usize)> = edges.iter().map(|&(from, to, _)| (from, to)).collect(); let components = strongly_connected_components(nodes.len(), &plain); - milp.builder.scope = Scope::SharedContexts; + milp.builder.scope = Scope::MaterializedContexts; let mut rank: HashMap = HashMap::new(); for (from, to, active) in edges { let size = components.size_of(from); @@ -1020,16 +1026,16 @@ pub(crate) fn strongly_connected_components(count: usize, edges: &[(usize, usize Components { of, sizes } } -/// The shareable candidates in order: each before every shareable +/// The materialized candidates in order: each before every materialized /// candidate its context may read (readers first), so `a[q, m]` exists /// before the reach rows of what `m` reads. A cycle among contexts is /// broken at the first candidate met. -pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec { +pub(crate) fn materialized_readers_first(milp: &SelectionMilp<'_>) -> Vec { let inputs = milp.inputs; let mut order = Vec::new(); let mut state: BTreeMap = BTreeMap::new(); // Post-order over "m reads c" (children = what m reads), reversed. - for &start in inputs.context_of_shareable.keys() { + for &start in inputs.context_of_materialized.keys() { if state.contains_key(&start) { continue; } @@ -1045,7 +1051,7 @@ pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec) -> Vec) { let inputs = milp.inputs; - milp.builder.scope = Scope::SharedContexts; - let order = shareables_readers_first(milp); - // Which queries may depend on each shareable candidate: read in their + milp.builder.scope = Scope::MaterializedContexts; + let order = materialized_readers_first(milp); + // Which queries may depend on each materialized candidate: read in their // context, or in the context of one they may depend on. let mut may_reach: BTreeMap> = BTreeMap::new(); for query in 0..milp.query_count() { @@ -1073,9 +1079,9 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { } } } - for &shareable in &order { - let queries = may_reach.get(&shareable).cloned().unwrap_or_default(); - let context = inputs.context_of_shareable[&shareable]; + for &materialized in &order { + let queries = may_reach.get(&materialized).cloned().unwrap_or_default(); + let context = inputs.context_of_materialized[&materialized]; for &option in &inputs.contexts[context].options { if inputs.kind[option.0] == RunKind::Materialized { may_reach @@ -1085,25 +1091,30 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { } } } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { + for &materialized in &order { + for &query in may_reach.get(&materialized).into_iter().flatten() { let reach = milp.builder.continuous("a (reach)", Some(1.0)); - milp.reach.insert((query, shareable), reach); + milp.reach.insert((query, materialized), reach); } } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - if let Some(&direct) = milp.choice[query].get(&shareable) { + for &materialized in &order { + for &query in may_reach.get(&materialized).into_iter().flatten() { + let reach = milp.reach[&(query, materialized)]; + if let Some(&direct) = milp.choice[query].get(&materialized) { milp.builder .row("6 reach from below", constraint!(reach >= direct)); } - for &reader in milp.shared_readers.get(&shareable).into_iter().flatten() { + for &reader in milp + .materialized_readers + .get(&materialized) + .into_iter() + .flatten() + { let Some(&above) = milp.reach.get(&(query, reader)) else { continue; }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; + let context = inputs.context_of_materialized[&reader]; + let read = milp.choice[context][&materialized]; milp.builder.row( "6 reach from below", constraint!(reach >= above + read - 1.0), @@ -1112,20 +1123,20 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { } } // From above, where needed. - let exact = shareables_needing_exact_reach(milp); - for &shareable in &order { - if !exact.contains(&shareable) { + let exact = materialized_needing_exact_reach(milp); + for &materialized in &order { + if !exact.contains(&materialized) { continue; } - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; + for &query in may_reach.get(&materialized).into_iter().flatten() { + let reach = milp.reach[&(query, materialized)]; let mut sources = Expression::from(0.0); - if let Some(&direct) = milp.choice[query].get(&shareable) { + if let Some(&direct) = milp.choice[query].get(&materialized) { sources += direct; } for &reader in milp - .shared_readers - .get(&shareable) + .materialized_readers + .get(&materialized) .cloned() .unwrap_or_default() .iter() @@ -1133,8 +1144,8 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { let Some(&above) = milp.reach.get(&(query, reader)) else { continue; }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; + let context = inputs.context_of_materialized[&reader]; + let read = milp.choice[context][&materialized]; let through = milp.builder.continuous("y (reach through)", Some(1.0)); milp.builder .row("6 reach from above", constraint!(through <= above)); @@ -1165,11 +1176,11 @@ pub(crate) fn reader_constraints_of_candidates<'a>( of_candidate } -/// The shareable candidates whose `a` must be exact from above: those a +/// The materialized candidates whose `a` must be exact from above: those a /// counting reader constraint (minimum readers, highest shared /// materialization) names or whose context holds a constrained candidate, -/// and transitively every shareable candidate whose context reads one. -fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet { +/// and transitively every materialized candidate whose context reads one. +fn materialized_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet { let inputs = milp.inputs; let constraints = reader_constraints_of_candidates(milp); let mut stack: Vec = Vec::new(); @@ -1188,7 +1199,12 @@ fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet) -> BTreeSet]) -> /// Row 9: reader constraints, on every node of the constrained candidate. /// A node in a query's context is reached by that query alone; a node in -/// shareable `m`'s context, and `m` itself, by the queries that depend on +/// materialized `m`'s context, and `m` itself, by the queries that depend on /// `m`. Reads into an `I_r` and per-output-reference reach inside one query /// are not modeled: the constraint is then applied to every reference of /// the query (conservative, counted as inexact). @@ -1247,7 +1268,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { (count, most) }; for (candidate, constraints) in constraints { - let shareable = inputs.kind[candidate.0] == RunKind::Materialized; + let materialized = inputs.kind[candidate.0] == RunKind::Materialized; // Its uses: (context, p) per context holding it as an option it // computes. let uses: Vec<(usize, Variable)> = (0..milp.choice.len()) @@ -1267,7 +1288,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { } => { let queries = bucket_queries(&query_of_reference, query_outputs); let minimum = *minimum as f64; - if shareable { + if materialized { if let Some(&selected) = milp.selected.get(&candidate) { let (count, _) = reach_count(milp, &queries, candidate); milp.builder.row( @@ -1293,7 +1314,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { } ReaderConstraint::HighestSharedOneEvaluation { query_outputs, .. } => { let queries = bucket_queries(&query_of_reference, query_outputs); - if shareable { + if materialized { let Some(&selected) = milp.selected.get(&candidate) else { continue; }; @@ -1308,13 +1329,13 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { } } for &reader in milp - .shared_readers + .materialized_readers .get(&candidate) .cloned() .unwrap_or_default() .iter() { - let context = inputs.context_of_shareable[&reader]; + let context = inputs.context_of_materialized[&reader]; let read = milp.choice[context][&candidate]; let (count, most) = reach_count(milp, &queries, reader); let single = milp.builder.binary("g (single-query readers)"); @@ -1376,7 +1397,7 @@ fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { forbid_reach(milp, left, candidate, &uses); continue; } - if shareable { + if materialized { if let (Some(&l), Some(&r)) = ( milp.reach.get(&(left, candidate)), milp.reach.get(&(right, candidate)), @@ -1481,7 +1502,7 @@ fn context_work(milp: &mut SelectionMilp<'_>, context: usize) -> (Expression, f6 /// Row 7: each query's latency, `asap_cost::plan_cost`'s definition: its /// request time, plus the work of every query-time run it reaches, each -/// once (its own context's, and each query-time shareable candidate's run +/// once (its own context's, and each query-time materialized candidate's run /// and context through `T[q, m]`), plus the longest compaction among the /// ingestion-time states it reaches. fn latency_rows(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { @@ -1502,20 +1523,20 @@ fn latency_rows(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> } let unit = milp.ms_per_latency_unit; let request = request_ms / unit; - // `L[m]` per query-time shareable candidate. - milp.builder.scope = Scope::SharedContexts; + // `L[m]` per query-time materialized candidate. + milp.builder.scope = Scope::MaterializedContexts; let mut shared_latency: BTreeMap = BTreeMap::new(); - let shareables: Vec<(PhysicalCandidateId, usize)> = inputs - .context_of_shareable + let materialized_candidates: Vec<(PhysicalCandidateId, usize)> = inputs + .context_of_materialized .iter() .map(|(&m, &context)| (m, context)) .collect(); - for (m, context) in shareables { + for (m, context) in materialized_candidates { if !milp.runs_at_query_time(context) { continue; } let (work, most) = context_work(milp, context); - milp.builder.scope = Scope::SharedContexts; + milp.builder.scope = Scope::MaterializedContexts; if most <= 0.0 { continue; } @@ -1599,7 +1620,7 @@ fn accuracy_rows(milp: &mut SelectionMilp<'_>) { // (candidate, dep, parts) per run `q` may depend on. let mut counted: Vec<(PhysicalCandidateId, Expression, usize)> = Vec::new(); // Where each counted run gives its value: (context, slot), `None` - // for a shareable candidate's own run. One value per place. + // for a materialized candidate's own run. One value per place. let mut places: BTreeSet<(usize, Option)> = BTreeSet::new(); for &(context, candidate) in &approximate_runs { let Some(weight) = inputs.accuracy_weights[query].get(&candidate).copied() else { @@ -1719,7 +1740,7 @@ pub(crate) struct IndependentPart { /// The independent parts of `milp`, in order of their first query (the /// part without a query last). Queries fall into one part when they may -/// share a shareable candidate, a run of an `I_r`, a source's raw retention or +/// share a materialized candidate, a run of an `I_r`, a source's raw retention or /// anything else a row joins. pub(crate) fn independent_parts(milp: &SelectionMilp<'_>) -> Vec { let column_of: HashMap = milp diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs index b7c6971d5..c59c2c3b4 100644 --- a/crates/plan-selection/src/milp/model/tests.rs +++ b/crates/plan-selection/src/milp/model/tests.rs @@ -297,7 +297,7 @@ fn candidates_where( .collect() } -/// Adds `Σ x[m] ≥ 1` over the shareable `candidates`: one is selected. +/// Adds `Σ x[m] ≥ 1` over the materialized `candidates`: one is selected. fn require_one_of(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { let sum: Expression = candidates .iter() @@ -478,7 +478,7 @@ fn a_kept_pane_chain_keeps_the_longest_readers_span() { node.asap(), Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) ) && (candidate.properties.timing() == ExecutionTiming::IngestionTime - && candidate.properties.lifetime().is_materialized()) + && candidate.properties.is_materialized()) }); require_one_of(milp, &chains); }) @@ -564,7 +564,7 @@ fn one_ingestion_scan_shared_by_two_sketch_configurations_is_charged_once() { let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { let sketches = candidates_where(milp, |candidate, node| { candidate.properties.timing() == ExecutionTiming::IngestionTime - && candidate.properties.lifetime().is_materialized() + && candidate.properties.is_materialized() && matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { .. }) @@ -616,11 +616,11 @@ fn ingestion_scans_under_refresh_windows( let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { let states = candidates_where(milp, |candidate, _| { candidate.properties.timing() == ExecutionTiming::IngestionTime - && candidate.properties.lifetime().is_materialized() + && candidate.properties.is_materialized() }); let window_of = |state: PhysicalCandidateId| { milp.inputs - .ingestion_context_of_shareable + .ingestion_context_of_materialized .get(&state) .map(|&context| milp.inputs.contexts[context].kind) }; @@ -805,7 +805,7 @@ fn a_stateless_subtree_two_state_nodes_read_runs_once() { // Structure // --------------------------------------------------------------------------- -// "Used only if read": a query context may not select a shareable +// "Used only if read": a query context may not select a materialized // candidate no used reader there reads, so a phantom read cannot satisfy // `MinimumQueryReaders`. With every reader of a two-reader `OneEvaluation` // result in the second query's context forbidden, selecting it is @@ -836,7 +836,9 @@ fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { }) .filter(|&m| milp.selected.contains_key(&m) && milp.choice[1].contains_key(&m)) .collect(); - let m = *constrained.first().expect("a two-reader shareable result"); + let m = *constrained + .first() + .expect("a two-reader materialized result"); let selected = milp.selected[&m]; milp.builder.row("test", constraint!(selected >= 1.0)); let readers: Vec = milp.readers[1][&m] @@ -863,9 +865,9 @@ fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { } // A not-materialized query-time candidate selected in two contexts is two -// nodes; a shareable candidate two queries read is one node. +// nodes; a materialized candidate two queries read is one node. #[test] -fn runs_decode_one_node_per_context_and_one_per_shareable_candidate() { +fn runs_decode_one_node_per_context_and_one_per_materialized_candidate() { let fixture = through_stages( &[query("sum(m)", 60_000), query("sum(m) * 2", 60_000)], true, @@ -873,7 +875,7 @@ fn runs_decode_one_node_per_context_and_one_per_shareable_candidate() { let unshared = solve_and_check(&fixture, &cost_inputs(), |milp| { let shared = candidates_where(milp, |candidate, _| { candidate.properties.timing() == ExecutionTiming::QueryTime - && candidate.properties.lifetime().is_materialized() + && candidate.properties.is_materialized() }); forbid(milp, &shared); let ingestion = candidates_where(milp, |candidate, _| { diff --git a/crates/plan-selection/src/milp/warm_start.rs b/crates/plan-selection/src/milp/warm_start.rs index 6a432cc85..4e497bd18 100644 --- a/crates/plan-selection/src/milp/warm_start.rs +++ b/crates/plan-selection/src/milp/warm_start.rs @@ -8,7 +8,7 @@ //! plus its inputs' cheapest subtrees), unless the context already chose a //! candidate of that slot, which every reader must then accept (one //! candidate per slot per context); a dead end backtracks to the next -//! candidate. A shareable candidate's context is derived once, when it is +//! candidate. A materialized candidate's context is derived once, when it is //! first selected; a candidate of an `I_r` is chosen in the `I_r` of the //! reading context's refresh window. //! @@ -19,13 +19,13 @@ //! planned again by latency alone. //! //! **Greedy sharing.** A second plan prices what the queries planned -//! before already selected (a shareable candidate, a candidate of an `I_r`) at +//! before already selected (a materialized candidate, a candidate of an `I_r`) at //! zero, so later queries read it. Both plans, in workload order and //! reversed, are costed exactly by the MILP's own rows and objective //! ([`Completion`]); the cheapest feasible one is the start. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use asap_cost::execution_demand::evaluation_rate; +use asap_cost::execution_demand::{evaluation_rate, materialized_run_rate, MaterializedRunRate}; use asap_types::stages::{output_references, PhysicalCandidateId, ReaderConstraint}; use good_lp::Variable; @@ -65,7 +65,7 @@ enum Approximate { } /// A plan under construction: per context, the candidate of each slot, and -/// the selected shareable candidates. +/// the selected materialized candidates. #[derive(Debug, Clone, Default)] struct Plan { chosen: Vec>, @@ -106,7 +106,7 @@ pub(crate) fn warm_start( orders.push(order); } // (share, amortized): alone; reading what earlier queries selected; and - // that with each shareable candidate's own cost divided among the + // that with each materialized candidate's own cost divided among the // queries that may reach it. let strategies = [(false, false), (true, false), (true, true)]; let mut tried = 0; @@ -127,7 +127,7 @@ pub(crate) fn warm_start( if let Ok(completed) = completed { debug(|| { format!( - "warm start: plan (share {share}, amortized {amortized}) costs {:e}, {} shareable selected", + "warm start: plan (share {share}, amortized {amortized}) costs {:e}, {} materialized selected", completed.cost_per_second, plan.selected.len() ) @@ -144,12 +144,12 @@ pub(crate) fn warm_start( } } } - // Greedy sharing: prefer one more shareable candidate that several + // Greedy sharing: prefer one more materialized candidate that several // queries may read (most readers first), kept when the exact cost // falls. let mut sharable: Vec<(usize, PhysicalCandidateId)> = milp .inputs - .context_of_shareable + .context_of_materialized .keys() .map(|&m| (estimates.readers[m.0], m)) .filter(|&(readers, _)| readers >= 2) @@ -201,8 +201,8 @@ fn binaries_of(milp: &SelectionMilp<'_>, plan: &Plan) -> Vec<(Variable, f64)> { } } } - for shareable in &plan.selected { - if let Some(&variable) = milp.selected.get(shareable) { + for materialized in &plan.selected { + if let Some(&variable) = milp.selected.get(materialized) { binaries.push((variable, 1.0)); } } @@ -324,8 +324,8 @@ fn undo_all(plan: &mut Plan, undo: Vec) { Undo::Chose(context, slot) => { plan.chosen[context].remove(&slot); } - Undo::Selected(shareable) => { - plan.selected.remove(&shareable); + Undo::Selected(materialized) => { + plan.selected.remove(&materialized); } Undo::SlotRead(context, reader, input) => { plan.slot_reads.remove(&(context, reader, input)); @@ -342,7 +342,7 @@ struct Estimates { rate: Vec, /// Per candidate, its numbers (first costed context). numbers: Vec>, - /// Per candidate, the queries that may depend on it (shareable ones). + /// Per candidate, the queries that may depend on it (materialized ones). readers: Vec, } @@ -357,6 +357,20 @@ impl Estimates { .unwrap_or(1.0) }) .collect(); + // Each query's requests per second: range steps not counted. + let request_rate: Vec = milp + .entries + .iter() + .map(|entry| evaluation_rate([(&entry.recurrence, None)], horizon).unwrap_or(1.0)) + .collect(); + // The slowest of `rates` over the queries that may reach `m`. + let slowest = |m, rates: &[f64]| { + milp.queries_reaching(m) + .iter() + .map(|&query| rates[query]) + .fold(f64::INFINITY, f64::min) + .min(1e9) + }; let rate = milp .inputs .contexts @@ -364,15 +378,17 @@ impl Estimates { .map(|context| match context.kind { ContextKind::Query(query) => query_rate[query], ContextKind::Ingestion(_) => 1.0, - ContextKind::Materialized(m) if context.at_query_time => { - let readers = milp.queries_reaching(m); - readers - .iter() - .map(|&query| query_rate[query]) - .fold(f64::INFINITY, f64::min) - .min(1e9) + // As `context_rates` classifies `κ_m`. + ContextKind::Materialized(m) => { + match materialized_run_rate(milp.physical.candidates[m.0].properties) { + Some(MaterializedRunRate::PerSecondOfArrivingData) => 1.0, + Some(MaterializedRunRate::PerEvaluation) => slowest(m, &query_rate), + Some(MaterializedRunRate::PerRequest) => slowest(m, &request_rate), + // `κ_m` exists only for a materialized `m`; + // `context_rates` rejects any other. + None => 1.0, + } } - ContextKind::Materialized(_) => 1.0, }) .collect(); let numbers = milp @@ -382,7 +398,7 @@ impl Estimates { .map(|data| data.usage.as_ref().ok().map(NodeNumbers::of)) .collect(); let mut readers = vec![0; milp.inputs.candidate_data.len()]; - for &m in milp.inputs.context_of_shareable.keys() { + for &m in milp.inputs.context_of_materialized.keys() { readers[m.0] = milp.queries_reaching(m).len(); } Self { @@ -401,10 +417,10 @@ struct Derivation<'m, 'a> { objective: Objective, approximate: Approximate, share: bool, - /// Divide a shareable candidate's own cost among the queries that may + /// Divide a materialized candidate's own cost among the queries that may /// reach it. amortized: bool, - /// Shareable candidates estimated free to read (greedy sharing). + /// Materialized candidates estimated free to read (greedy sharing). preferred: &'m BTreeSet, undo: Vec, steps: usize, @@ -662,7 +678,7 @@ impl Derivation<'_, '_> { /// The runs query `query` depends on in `plan`: `(context, candidate)` for /// candidates computed in its context and in the contexts of the -/// shareable candidates it reaches, and each such shareable candidate +/// materialized candidates it reaches, and each such materialized candidate /// `(its context, itself)`. fn runs_reached( milp: &SelectionMilp<'_>, diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index 50ec26c93..4c26fc1f1 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -305,13 +305,6 @@ pub enum ErrorSelectingPlan { roots: usize, entries: usize, }, - /// A candidate that cannot be costed as a whole; the error names the node - /// (one whose timing and lifetime cannot go together: a Stage 2 bug). - #[error("candidate {candidate}: {error}")] - InvalidCandidate { - candidate: String, - error: AnalyticalCostError, - }, #[error("candidate {candidate}, node {node:?}: {error}")] Cost { candidate: String, diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index 789dfccd9..6ede3b230 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -18,18 +18,17 @@ //! (`satisfies`) and latency hold, or it is an error. //! //! With `ASAP_STAGE3_REPORT=`, one JSON line per run is appended -//! there: the model's size per family and per scope (the shareable +//! there: the model's size per family and per scope (the materialized //! candidates' contexts apart), the timings, the warm start's cost, the //! objective, `plan_cost` and the dual bound (measurement only). -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use std::time::Instant; use asap_cost::analytical::AnalyticalCostError; use asap_cost::{materialized_tumbling_window_chains, node_map, plan_cost, CostInputs}; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, @@ -62,20 +61,6 @@ pub(crate) fn cost_error( } } -/// `plan`'s [`materialized_tumbling_window_chains`], or -/// [`ErrorSelectingPlan::InvalidCandidate`] naming the node whose timing and -/// lifetime cannot go together. -pub(crate) fn materialized_chains_of( - plan: &PhysicalWorkloadPlan, -) -> Result, ErrorSelectingPlan> { - materialized_tumbling_window_chains(&plan.dag).map_err(|error| { - ErrorSelectingPlan::InvalidCandidate { - candidate: plan.id.clone(), - error, - } - }) -} - /// The largest relative difference between the objective and `plan_cost` /// that counts as equal. const COST_EQUALITY_TOLERANCE: f64 = 1e-6; @@ -157,7 +142,7 @@ pub fn select_plan( let (cost, _) = plan_cost( &plan.dag, &node_map(&plan.dag), - &materialized_chains_of(plan)?, + &materialized_tumbling_window_chains(&plan.dag).map_err(cost_error(&plan.id))?, &recurrence, &ranges, data, diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs index 2344bb0b8..7cc2daf94 100644 --- a/crates/plan-selection/src/selector.rs +++ b/crates/plan-selection/src/selector.rs @@ -16,10 +16,13 @@ use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; use crate::report::{ AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, }; -use crate::select::{cost_error, materialized_chains_of}; +use crate::select::cost_error; use crate::SelectionModels; use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{node_map, plan_cost, query_reach, tumbling_window_retention_ms}; +use asap_cost::{ + materialized_tumbling_window_chains, node_map, plan_cost, query_reach, + tumbling_window_retention_ms, +}; type Ranking = PlanRanking; use asap_types::workload::bounds::with_confirmed_weights; @@ -179,7 +182,8 @@ impl<'a> PlanSelector<'a> { return Ok(CalibrationFit::default()); } let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_chains_of(candidate)?; + let materialized_chains = materialized_tumbling_window_chains(&candidate.dag) + .map_err(cost_error(&candidate.id))?; let (_, fit) = plan_cost( &candidate.dag, &nodes, @@ -225,7 +229,8 @@ impl<'a> PlanSelector<'a> { } }; let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_chains_of(candidate)?; + let materialized_chains = materialized_tumbling_window_chains(&candidate.dag) + .map_err(cost_error(&candidate.id))?; let settings = self.models.cost_inputs; let (cost, _) = plan_cost( &candidate.dag, diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 38c1e958d..af2bc3cb9 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -1545,7 +1545,7 @@ impl OptimizationPass for StoredAsForced { let selected = output .physical_properties .get(&node) - .filter(|properties| properties.lifetime().is_materialized()); + .filter(|properties| properties.is_materialized()); let own = match node.asap() { Some( ASAPOp::SummaryAgg { .. } diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 893938139..db09e1640 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -79,6 +79,7 @@ use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::operator::operator_properties::Source; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; +use crate::physical::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is kept. @@ -400,7 +401,7 @@ impl PhysicalASAPDAG { if let Some(node) = self .nodes .iter() - .find(|node| node.lifetime.is_across_evaluations() && node.retention_ms.is_none()) + .find(|node| is_view(node) && node.retention_ms.is_none()) { return Err(InstallableDagError::ViewWithoutRetention { node_id: node.id }); } @@ -729,7 +730,7 @@ impl PhysicalASAPDAG { let views: HashSet = self .nodes .iter() - .filter(|node| node.lifetime.is_across_evaluations()) + .filter(|node| is_view(node)) .map(|node| node.id) .collect(); // The IR of each node, which coverage is derived from. @@ -763,7 +764,19 @@ impl PhysicalASAPDAG { { window_by_view.insert(id, window); } - let refreshed_at_ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + let refreshed_at_ingestion = match (node.output_state.timing, node.lifetime) { + ( + ExecutionTiming::IngestionTime, + Lifetime::None | Lifetime::AcrossEvaluations(_), + ) => true, + ( + ExecutionTiming::QueryTime, + Lifetime::None | Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), + ) => false, + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; if refreshed_at_ingestion { // An ingestion run reads each view for the window ending at // its end, so a view read must end where the raw windows do. @@ -1160,3 +1173,19 @@ fn with_replaced( } replace(node, old, new, &mut HashMap::new()) } + +/// Whether `node` is a materialized view: its output is materialized across +/// evaluations, at either timing. +fn is_view(node: &PhysicalASAPDAGNode) -> bool { + match (node.output_state.timing, node.lifetime) { + ( + ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(_), + ) => true, + (ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime, Lifetime::None) + | (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => false, + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + } +} diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 38856cc87..8677e9a2f 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -634,8 +634,20 @@ impl PhysicalASAPDAG { }, ); } + let materialized_across_evaluations = match (properties.timing(), properties.lifetime()) + { + ( + ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(_), + ) => true, + (ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime, Lifetime::None) + | (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => false, + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }; if node.retention_ms.is_some() - && !node.lifetime.is_across_evaluations() + && !materialized_across_evaluations && !super::materialized_view::is_query_time_scan(node) { return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs index cd9d2f704..d175a8700 100644 --- a/crates/types/src/ir/properties/timing.rs +++ b/crates/types/src/ir/properties/timing.rs @@ -63,7 +63,7 @@ use crate::ir::properties::execution_data_state::{ DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; use crate::ir::schema::{DataType, FieldDataType, Schema}; -use crate::physical::PhysicalProperties; +use crate::physical::{satisfies_physical, Evaluation, PhysicalProperties, PhysicalRequirement}; #[cfg(test)] use crate::physical::{Lifetime, Medium}; @@ -353,7 +353,10 @@ pub fn apply_physical_properties_to_roots( .get(node) .expect("written node is timed"); if properties.timing() == ExecutionTiming::IngestionTime - && !properties.readable_at(ExecutionTiming::QueryTime) + && !satisfies_physical( + &properties, + &read_in_the_same_evaluation(ExecutionTiming::QueryTime), + ) && primitive(node) == DataPrimitive::Raw { return Err(ExecutionDataStateError::UnstoredIngestionRoot { @@ -581,7 +584,7 @@ fn validate_asap( && avail.timing == ExecutionTiming::IngestionTime && !physical_properties .get(child) - .is_some_and(|p| p.readable_at(timing)) + .is_some_and(|p| satisfies_physical(&p, &read_in_the_same_evaluation(timing))) { return Err( ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { @@ -734,7 +737,7 @@ fn validate_non_asap( // materialized at ingestion time. let reads_stored = physical_properties .get(child) - .is_some_and(|p| p.readable_at(timing)); + .is_some_and(|p| satisfies_physical(&p, &read_in_the_same_evaluation(timing))); if (s.timing != timing && !reads_stored) || (s.primitive != DataPrimitive::Raw && !passes_state) { @@ -865,6 +868,13 @@ fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { } } +/// r^phys of a reader at `timing` that reads its input's output in the same +/// evaluation (or ingestion work), as every edge of the timing pass does. +fn read_in_the_same_evaluation(timing: ExecutionTiming) -> PhysicalRequirement { + PhysicalRequirement::new(timing, Evaluation::Same) + .expect("a same-evaluation read is valid at either timing") +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index 932f6e550..0a417ff4a 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,11 +1,15 @@ //! #509 Stage 2 decision data: the physical properties of each candidate. //! //! - [`properties`] — when a candidate runs and how long its output lives -//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]). +//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]), and what a reader +//! requires of them ([`PhysicalRequirement`], [`satisfies_physical`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). pub mod properties; pub mod workload_plan; -pub use properties::{InvalidPhysicalProperties, Lifetime, Medium, PhysicalProperties}; +pub use properties::{ + satisfies_physical, Evaluation, InvalidPhysicalProperties, InvalidPhysicalRequirement, + Lifetime, Medium, PhysicalProperties, PhysicalRequirement, +}; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index bf82abbb1..8260bf6e6 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -33,20 +33,6 @@ pub enum Lifetime { AcrossEvaluations(Medium), } -impl Lifetime { - /// Whether the output outlives its reader: `OneEvaluation` or - /// `AcrossEvaluations`. - pub fn is_materialized(self) -> bool { - self != Self::None - } - - /// Whether the output is materialized across evaluations: a materialized view - /// that later evaluations reuse. - pub fn is_across_evaluations(self) -> bool { - matches!(self, Self::AcrossEvaluations(_)) - } -} - /// A timing and a lifetime that cannot go together. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InvalidPhysicalProperties { @@ -124,26 +110,15 @@ impl PhysicalProperties { self.lifetime } - /// Whether a reader running at `reader` may read this output: one at the - /// same timing, or a query-time reader of an ingestion-time output materialized - /// until a query reads it. - pub fn readable_at(self, reader: ExecutionTiming) -> bool { + /// Whether the output outlives its reader (lifetime `OneEvaluation` or + /// `AcrossEvaluations`): materialized(v) ⇔ ℓ_v ≠ None. + pub fn is_materialized(self) -> bool { use ExecutionTiming::{IngestionTime, QueryTime}; - match (self.timing, self.lifetime, reader) { - // Runs inside the execution that reads it: only a reader at its - // own timing. - (QueryTime, Lifetime::None, QueryTime) => true, - (QueryTime, Lifetime::None, IngestionTime) => false, - (IngestionTime, Lifetime::None, IngestionTime) => true, - (IngestionTime, Lifetime::None, QueryTime) => false, - // Kept until a query reads it: ingestion work and queries both read it. - (IngestionTime, Lifetime::AcrossEvaluations(_), IngestionTime | QueryTime) => true, - // A query-time output: queries read it; ingestion work never does. - (QueryTime, Lifetime::OneEvaluation(_), QueryTime) => true, - (QueryTime, Lifetime::OneEvaluation(_), IngestionTime) => false, - (QueryTime, Lifetime::AcrossEvaluations(_), QueryTime) => true, - (QueryTime, Lifetime::AcrossEvaluations(_), IngestionTime) => false, - (IngestionTime, Lifetime::OneEvaluation(_), _) => { + match (self.timing, self.lifetime) { + (QueryTime | IngestionTime, Lifetime::None) => false, + (QueryTime, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_)) + | (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::OneEvaluation(_)) => { unreachable!("rejected by PhysicalProperties::new") } } @@ -169,6 +144,103 @@ impl PhysicalProperties { } } +/// Which evaluation's output a reader reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Evaluation { + /// The output of the evaluation the reader runs in (or, at ingestion + /// time, of the ingestion work it runs in). + Same, + /// The output of an earlier evaluation, reused by a later one. + Later, +} + +/// A reader timing and an evaluation that cannot go together. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum InvalidPhysicalRequirement { + #[error("an ingestion-time reader cannot read a later evaluation's output: there is no evaluation at ingestion time")] + LaterEvaluationAtIngestion, +} + +/// r^phys_{c,i}: when a consumer reads one input, and from which +/// evaluation. The reader's schedule comes with the schedule σ (#1200 PR 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalRequirement { + reader_timing: ExecutionTiming, + reads_from: Evaluation, +} + +impl PhysicalRequirement { + /// The requirement of a reader at `reader_timing` reading `reads_from`, + /// unless they cannot go together. + pub fn new( + reader_timing: ExecutionTiming, + reads_from: Evaluation, + ) -> Result { + use ExecutionTiming::{IngestionTime, QueryTime}; + match (reader_timing, reads_from) { + (IngestionTime, Evaluation::Later) => { + Err(InvalidPhysicalRequirement::LaterEvaluationAtIngestion) + } + (QueryTime | IngestionTime, Evaluation::Same) | (QueryTime, Evaluation::Later) => { + Ok(Self { + reader_timing, + reads_from, + }) + } + } + } + + pub fn reader_timing(self) -> ExecutionTiming { + self.reader_timing + } + + pub fn reads_from(self) -> Evaluation { + self.reads_from + } +} + +/// α^phys_{u,c,i}: whether a producer with `producer`'s properties may serve +/// an input with `requirement` (#1200, timing and lifetime table): +/// - at ingestion time: the producer runs at ingestion time; +/// - at query time, in the same evaluation: a query-time producer with any +/// lifetime, or an ingestion-time producer materialized across evaluations; +/// - at query time, in a later evaluation: a producer materialized across +/// evaluations. +pub fn satisfies_physical( + producer: &PhysicalProperties, + requirement: &PhysicalRequirement, +) -> bool { + use ExecutionTiming::{IngestionTime, QueryTime}; + match ( + producer.timing, + producer.lifetime, + requirement.reader_timing, + requirement.reads_from, + ) { + (IngestionTime, Lifetime::OneEvaluation(_), ..) => { + unreachable!("rejected by PhysicalProperties::new") + } + (_, _, IngestionTime, Evaluation::Later) => { + unreachable!("rejected by PhysicalRequirement::new") + } + // An ingestion-time reader reads what runs at ingestion time. + ( + IngestionTime, + Lifetime::None | Lifetime::AcrossEvaluations(_), + IngestionTime, + Evaluation::Same, + ) => true, + (QueryTime, _, IngestionTime, Evaluation::Same) => false, + // A query-time reader in the same evaluation. + (QueryTime, _, QueryTime, Evaluation::Same) => true, + (IngestionTime, Lifetime::AcrossEvaluations(_), QueryTime, Evaluation::Same) => true, + (IngestionTime, Lifetime::None, QueryTime, Evaluation::Same) => false, + // A later evaluation reads only what lives across evaluations. + (_, Lifetime::AcrossEvaluations(_), QueryTime, Evaluation::Later) => true, + (_, Lifetime::None | Lifetime::OneEvaluation(_), QueryTime, Evaluation::Later) => false, + } +} + #[cfg(test)] mod tests { use super::*; @@ -221,48 +293,97 @@ mod tests { } } - /// `readable_at` for every valid state and both reader timings. + /// `satisfies_physical` for every valid state and every valid requirement. #[test] - fn readable_at_over_every_valid_state() { + fn satisfies_physical_over_every_valid_state() { use ExecutionTiming::{IngestionTime, QueryTime}; let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); - // (state, readable by a query-time reader, readable by an ingestion-time reader) + let same_at_query = PhysicalRequirement::new(QueryTime, Evaluation::Same).unwrap(); + let same_at_ingestion = PhysicalRequirement::new(IngestionTime, Evaluation::Same).unwrap(); + let later_at_query = PhysicalRequirement::new(QueryTime, Evaluation::Later).unwrap(); + // (state, query time same evaluation, ingestion time, later evaluation) let mut expected = Vec::new(); for medium in [Medium::Memory, Medium::Disk] { expected.extend([ - (at(QueryTime, Lifetime::OneEvaluation(medium)), true, false), + ( + at(QueryTime, Lifetime::OneEvaluation(medium)), + true, + false, + false, + ), ( at(QueryTime, Lifetime::AcrossEvaluations(medium)), true, false, + true, ), ( at(IngestionTime, Lifetime::AcrossEvaluations(medium)), true, true, + true, ), ]); } expected.extend([ - (at(QueryTime, Lifetime::None), true, false), - (at(IngestionTime, Lifetime::None), false, true), + (at(QueryTime, Lifetime::None), true, false, false), + (at(IngestionTime, Lifetime::None), false, true, false), ]); assert_eq!(expected.len(), PhysicalProperties::every_valid().len()); for properties in PhysicalProperties::every_valid() { - let (_, at_query, at_ingestion) = expected + let (_, query, ingestion, later) = expected .iter() .find(|(state, ..)| *state == properties) .expect("every valid state has an expectation"); assert_eq!( - properties.readable_at(QueryTime), - *at_query, + satisfies_physical(&properties, &same_at_query), + *query, + "{properties:?}" + ); + assert_eq!( + satisfies_physical(&properties, &same_at_ingestion), + *ingestion, + "{properties:?}" + ); + assert_eq!( + satisfies_physical(&properties, &later_at_query), + *later, "{properties:?}" ); + } + } + + /// `is_materialized` is lifetime ≠ None for every valid state. + #[test] + fn is_materialized_over_every_valid_state() { + for properties in PhysicalProperties::every_valid() { assert_eq!( - properties.readable_at(IngestionTime), - *at_ingestion, + properties.is_materialized(), + properties.lifetime() != Lifetime::None, "{properties:?}" ); } } + + /// An ingestion-time reader cannot read a later evaluation; every other + /// requirement is valid. + #[test] + fn later_evaluation_at_ingestion_is_the_invalid_requirement() { + use ExecutionTiming::{IngestionTime, QueryTime}; + assert_eq!( + PhysicalRequirement::new(IngestionTime, Evaluation::Later), + Err(InvalidPhysicalRequirement::LaterEvaluationAtIngestion) + ); + for (timing, evaluation) in [ + (QueryTime, Evaluation::Same), + (QueryTime, Evaluation::Later), + (IngestionTime, Evaluation::Same), + ] { + let requirement = PhysicalRequirement::new(timing, evaluation).unwrap(); + assert_eq!( + (requirement.reader_timing(), requirement.reads_from()), + (timing, evaluation) + ); + } + } } diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs index ac325e98c..d2a0ac253 100644 --- a/crates/types/src/physical/workload_plan.rs +++ b/crates/types/src/physical/workload_plan.rs @@ -29,6 +29,6 @@ impl PhysicalWorkloadPlan { self.dag .nodes .iter() - .any(|node| node.lifetime.is_materialized()) + .any(|node| node.lifetime != crate::physical::Lifetime::None) } } diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 911140f33..0e37e1f73 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -197,7 +197,7 @@ fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { })) .unwrap(); let mut assignment = PhysicalPropertiesAssignment::default(); - if lifetime.is_materialized() { + if lifetime != Lifetime::None { assignment.set(&chain, PhysicalProperties::new(timing, lifetime).unwrap()); } let physical_properties = apply_physical_properties(&estimate, &assignment).unwrap(); diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index f59566dcd..2140e3e9a 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -78,7 +78,7 @@ contains the candidate and that Stage 3 prices it. 1. Add the `Lifetime` variant or the valid (timing, lifetime) pair (`crates/types/src/physical/properties.rs`, `PhysicalProperties::new` and - `readable_at`) and its placement in + `satisfies_physical`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the `MaterializedOutputCandidate` options), after the existing options so default From 198b9a9f12df964bf94fab1e4ac91d94c7ff6e40 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:52:44 +0000 Subject: [PATCH 3/3] refactor: no interim abstractions in the physical properties types User decisions (2026-11-05). PhysicalRequirement is { consumer_timing, reads_from: Evaluation, consumer_schedule: Option }; EvaluationSchedule { interval_ms, phase_ms } is the schedule of the consumer's context (I_kappa, phi_kappa), None until the schedule sigma. Names this stack introduces say consumer, not reader (materialized_consumers, materialized_consumers_first). The five named PhysicalProperties consts and not_materialized_at are deleted; callers build PhysicalProperties::new(timing, lifetime). InvalidPhysicalRequirement folds into InvalidPhysicalProperties (LaterEvaluationAtIngestion). Cost reports an invalid pair through the export's PhysicalASAPDAGValidationError::InvalidPhysicalProperties, wrapped in AnalyticalCostError::InvalidPhysicalDag, instead of a parallel variant. Inlined: PhysicalASAPDAGNode::physical_properties(), cost_error, lists_every_node_of and its debug assertion; the tumbling-window-chain predicate is private to asap-cost. Test-only helpers inlined: PhysicalProperties::every_valid (tests enumerate the pairs new accepts), default_physical_properties and physical_properties_at_ingestion. Co-Authored-By: Claude Opus 5.5 --- crates/cost/src/analytical.rs | 11 +- crates/cost/src/execution_demand.rs | 32 ++- crates/cost/src/lib.rs | 8 +- crates/cost/src/usage.rs | 18 +- crates/cost/tests/ingestion_refresh_runs.rs | 69 ++++- crates/devtools/src/candidate_labels.rs | 23 +- crates/devtools/tests/asap_planner.rs | 12 +- crates/executor/tests/every_valid_state.rs | 27 +- .../executor/tests/weighted_topk_binding.rs | 9 +- crates/integration-tests/src/lib.rs | 48 ++-- .../tests/exact_composition.rs | 32 ++- .../tests/physical_properties_e2e.rs | 6 +- .../tests/promql_to_post_asap.rs | 12 +- .../tests/tumbling_window_merge.rs | 23 +- .../src/catalog/configurations.rs | 71 ++++- .../src/catalog/realize.rs | 11 +- crates/physical-asap-optimization/src/lib.rs | 40 ++- .../src/split_by_timing.rs | 20 +- crates/plan-selection/src/groups.rs | 24 +- crates/plan-selection/src/groups/tests.rs | 28 +- crates/plan-selection/src/lib.rs | 98 +++++-- crates/plan-selection/src/milp/cost_terms.rs | 2 +- crates/plan-selection/src/milp/model.rs | 20 +- crates/plan-selection/src/milp/model/tests.rs | 2 +- crates/plan-selection/src/select.rs | 28 +- crates/plan-selection/src/selector.rs | 33 ++- crates/planner/src/installable.rs | 14 +- crates/planner/src/run.rs | 4 - crates/planner/tests/runtime_install.rs | 21 +- .../src/ir/operator/tumbling_window_chain.rs | 8 +- crates/types/src/ir/physical_export.rs | 25 +- crates/types/src/ir/properties/timing.rs | 146 +++++++---- crates/types/src/physical/mod.rs | 6 +- crates/types/src/physical/properties.rs | 248 +++++++----------- crates/types/tests/materialized_view.rs | 45 +++- crates/types/tests/physical_export.rs | 28 +- crates/types/tests/schema_rebuilding.rs | 4 +- crates/types/tests/summary_merge_structure.rs | 8 +- 38 files changed, 841 insertions(+), 423 deletions(-) diff --git a/crates/cost/src/analytical.rs b/crates/cost/src/analytical.rs index dd136576f..9c740b18d 100644 --- a/crates/cost/src/analytical.rs +++ b/crates/cost/src/analytical.rs @@ -1489,13 +1489,10 @@ pub enum AnalyticalCostError { UnsupportedWithoutDistribution(&'static str), #[error("analytical arithmetic overflowed")] Overflow, - /// A node whose timing and lifetime cannot go together: a planner bug. - #[error("node {node:?}: {reason}")] - InvalidPhysicalProperties { - node: asap_types::ir::physical_export::PhysicalASAPNodeId, - #[source] - reason: asap_types::physical::InvalidPhysicalProperties, - }, + /// The physical DAG fails its validation (e.g. a node whose timing and + /// lifetime cannot go together): a planner bug. + #[error(transparent)] + InvalidPhysicalDag(asap_types::ir::physical_export::PhysicalASAPDAGValidationError), #[error("candidate has no supported exact or sketch state")] UnsupportedCandidate, #[error("inconsistent operator statistics: {0}")] diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/execution_demand.rs index 0b6700703..56bd48cd5 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/execution_demand.rs @@ -8,6 +8,7 @@ //! to price once per request, as `computations` does. use super::analytical::AnalyticalCostError; +use asap_types::ir::physical_export::PhysicalASAPDAGValidationError; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::ir::properties::ExecutionTiming; use asap_types::physical::{Lifetime, PhysicalProperties}; @@ -406,12 +407,15 @@ pub fn computations( let range_of = |query: usize| range_evaluations.get(query).copied().flatten(); let mut materialized_nodes: HashMap = HashMap::new(); for node in &dag.nodes { - let properties = node.physical_properties().map_err(|reason| { - AnalyticalCostError::InvalidPhysicalProperties { - node: node.id, - reason, - } - })?; + let properties = + PhysicalProperties::new(node.output_state.timing, node.lifetime).map_err(|reason| { + AnalyticalCostError::InvalidPhysicalDag( + PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: node.id, + reason, + }, + ) + })?; if let Some(rate) = materialized_run_rate(properties) { materialized_nodes.insert(node.id, rate); } @@ -719,7 +723,21 @@ mod tests { ), ]); } - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let (_, rate) = expected .iter() .find(|(state, _)| *state == properties) diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index ddf494add..f921d3bb7 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -41,10 +41,10 @@ pub use measured::{ pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, is_query_time_materialized_tumbling_window_chain, maintained_window_ms, - materialized_tumbling_window_chains, node_map, node_usage, plan_cost, query_reach, - retained_tumbling_windows, trailing_window, tumbling_window_retention_ms, - tumbling_window_retention_ms_of_plan, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, + family_name, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, + plan_cost, query_reach, retained_tumbling_windows, trailing_window, + tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, + NodeMap, NodeUsage, ProducerOutput, }; /// Cost per second of wall time; one unit is one CPU-millisecond under diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 9c9b9c0e8..903bb6111 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -4,6 +4,7 @@ //! whose module doc, "Cost model", describes the model), so Stage 3 computes //! no cost of its own. +use asap_types::ir::physical_export::PhysicalASAPDAGValidationError; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use crate::analytical::{ @@ -1656,7 +1657,7 @@ pub fn materialized_tumbling_window_chains( /// retained-window options (`retained_pane_options`) and /// [`tumbling_window_retention_ms`] cover both timings, since a chain materialized /// at either timing retains windows. -pub fn is_query_time_materialized_tumbling_window_chain( +fn is_query_time_materialized_tumbling_window_chain( node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, ) -> Result { let chain = matches!( @@ -1686,12 +1687,15 @@ pub fn is_query_time_materialized_tumbling_window_chain( fn timing_and_lifetime( node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, ) -> Result<(ExecutionTiming, Lifetime), AnalyticalCostError> { - let properties: PhysicalProperties = node.physical_properties().map_err(|reason| { - AnalyticalCostError::InvalidPhysicalProperties { - node: node.id, - reason, - } - })?; + let properties: PhysicalProperties = + PhysicalProperties::new(node.output_state.timing, node.lifetime).map_err(|reason| { + AnalyticalCostError::InvalidPhysicalDag( + PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: node.id, + reason, + }, + ) + })?; Ok((properties.timing(), properties.lifetime())) } diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs index 75d76f92d..5f9493ef4 100644 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ b/crates/cost/tests/ingestion_refresh_runs.rs @@ -3,6 +3,9 @@ //! a runtime refreshes the views of one refresh window in one ingestion run //! (issue #1151). +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::rc::Rc; use std::time::Duration; @@ -146,7 +149,13 @@ fn a_scan_under_two_chains_of_one_tumbling_window_width_is_computed_once() { chain(&scan, SketchAlgorithm::Kll, 10_000, 6), chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!(scan_computations(&dag), (1, 1.0)); } @@ -162,7 +171,13 @@ fn a_scan_under_chains_of_two_tumbling_window_widths_is_computed_for_each() { chain(&scan, SketchAlgorithm::Kll, 10_000, 6), chain(&scan, SketchAlgorithm::DDSketch, 60_000, 5), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!(scan_computations(&dag), (2, 2.0)); } @@ -212,7 +227,13 @@ fn a_scan_under_chains_of_10_10_and_60_s_is_computed_for_each_width() { chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), chain(&scan, SketchAlgorithm::Kll, 60_000, 5), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!(scan_computations(&dag), (2, 2.0)); } @@ -232,7 +253,13 @@ fn chains_of_one_width_and_two_phases_are_computed_apart() { 30, ), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!(scan_computations(&dag), (2, 2.0)); } @@ -247,7 +274,13 @@ fn nodes_below_a_shared_intermediate_node_are_computed_once_per_run() { chain(&shift, SketchAlgorithm::Kll, 10_000, 6), chain(&shift, SketchAlgorithm::DDSketch, 10_000, 30), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!(scan_computations(&dag), (1, 1.0)); assert_eq!( @@ -281,7 +314,13 @@ fn consumers_of_unbounded_windows_count_alone() { build(SketchAlgorithm::Kll), build(SketchAlgorithm::DDSketch), ], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); assert_eq!( computations_of(&dag, |op| matches!(op, NonASAPOp::Values { .. })), @@ -296,7 +335,13 @@ fn an_invalid_node_is_reported_not_skipped() { let scan = scan(); let mut dag = dag_of( &[chain(&scan, SketchAlgorithm::Kll, 10_000, 6)], - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION), + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ); let chain = dag .nodes @@ -309,10 +354,12 @@ fn an_invalid_node_is_reported_not_skipped() { assert_eq!( computations(&dag, &[], &[], 3600.0, None), Err( - asap_cost::analytical::AnalyticalCostError::InvalidPhysicalProperties { - node: chain, - reason: asap_types::physical::InvalidPhysicalProperties::OneEvaluationAtIngestion, - } + asap_cost::analytical::AnalyticalCostError::InvalidPhysicalDag( + asap_types::ir::physical_export::PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: chain, + reason: asap_types::physical::InvalidPhysicalProperties::OneEvaluationAtIngestion, + } + ) ) ); } diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 196a926d4..0e375f489 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -148,7 +148,10 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String let mut parts: Vec<(&str, Vec)> = Vec::new(); for node in &candidate.dag.nodes { let id = &node.id; - let Some(phase) = phase_label(node.physical_properties()) else { + let Some(phase) = phase_label(PhysicalProperties::new( + node.output_state.timing, + node.lifetime, + )) else { continue; }; let label = match &node.payload { @@ -194,7 +197,7 @@ fn phase_label( return Some("invalid timing and lifetime"); }; match (properties.timing(), properties.lifetime()) { - // Not materialized: no phase holds it; it runs inside its reader. + // Not materialized: no phase holds it; it runs inside its consumer. (QueryTime, Lifetime::None) => None, (IngestionTime, Lifetime::None) => None, (IngestionTime, Lifetime::AcrossEvaluations(_)) => Some("ingestion time"), @@ -586,7 +589,21 @@ mod tests { ), ]); } - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let (_, label) = expected .iter() .find(|(state, _)| *state == properties) diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 7bbed98db..8c4bae331 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -1,6 +1,9 @@ //! `asap_planner` writes valid stage documents: each stage's output over //! #509's examples and ad-hoc workloads, the deployment and cost inputs it //! plans with, and the inputs it refuses. +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -1604,8 +1607,13 @@ fn scans_projecting_different_labels_are_shared() { data, false, &mut |_, readers, options| { - let materialized = (readers.len() > 1) - .then_some(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION); + let materialized = (readers.len() > 1).then_some( + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ); options.iter().position(|option| *option == materialized) }, ) diff --git a/crates/executor/tests/every_valid_state.rs b/crates/executor/tests/every_valid_state.rs index 17f430664..42c7c8817 100644 --- a/crates/executor/tests/every_valid_state.rs +++ b/crates/executor/tests/every_valid_state.rs @@ -1,5 +1,6 @@ //! The executor's physical planner over every valid timing × lifetime //! state of one summary build. +use planner_types::physical::Medium; use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -65,7 +66,21 @@ fn plan() -> Rc { #[test] fn physical_planner_over_every_valid_state() { use ExecutionTiming::{IngestionTime, QueryTime}; - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let root = plan(); let mut assignment = PhysicalPropertiesAssignment::default(); let unmaterialized_at_ingestion = @@ -95,8 +110,14 @@ fn physical_planner_over_every_valid_state() { (IngestionTime, Lifetime::None) => { // Left unassigned: an unmaterialized build lands at query time. assert_eq!( - dag.nodes[2].physical_properties(), - Ok(PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME) + PhysicalProperties::new( + dag.nodes[2].output_state.timing, + dag.nodes[2].lifetime + ), + Ok( + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime") + ) ); } (IngestionTime, Lifetime::OneEvaluation(_)) => { diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 0264d7b6d..ce14b8dc5 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -13,8 +13,11 @@ use common::{compile_physical_asap_dag, computation_candidates}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; use planner_types::ir::properties::AccuracyTarget; +use planner_types::ir::properties::ExecutionTiming; use planner_types::ir::schema::*; use planner_types::ir::ASAPOp; +use planner_types::physical::Lifetime; +use planner_types::physical::Medium; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; // The heap candidates of `topk by(job)(2, sum by(service, job)(rate(m[1m])))` // rank the summed per-series rates of raw counter samples bound at the @@ -284,7 +287,11 @@ fn continuously_maintained_dag(candidate: &Rc) if matches!(node.operator, IrOperator::ASAP(ASAPOp::SummaryAgg { .. })) { assignment.set( &node, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index e8079bc70..537c19ffe 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -101,44 +101,30 @@ pub mod physical_export { use asap_types::ir::{apply_physical_properties, OperatorNode, PhysicalPropertiesAssignment}; use std::rc::Rc; - /// The physical properties of every node of `root` under the default - /// assignment (no sub-DAG materialized, so every summary computed at - /// query time). - pub fn default_physical_properties(root: &Rc) -> PhysicalPropertiesAssignment { - completed_physical_properties(root, &PhysicalPropertiesAssignment::default()) - } - - /// The physical properties of every node of `root` with every sub-DAG - /// materialized at ingestion time where the DAG allows it. - pub fn physical_properties_at_ingestion( - root: &Rc, - ) -> PhysicalPropertiesAssignment { - completed_physical_properties( - root, - &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( - std::slice::from_ref(root), - ), - ) - } - - fn completed_physical_properties( - root: &Rc, - assignment: &PhysicalPropertiesAssignment, - ) -> PhysicalPropertiesAssignment { - apply_physical_properties(root, assignment).expect("applying physical properties failed") - } - /// Time `root` (default assignment), then export the physical DAG. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(root, &default_physical_properties(root)) - .expect("physical ASAP DAG export failed") + compile_physical_asap_dag( + root, + &apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .expect("applying physical properties failed"), + ) + .expect("physical ASAP DAG export failed") } /// Time `root` with every sub-DAG materialized at ingestion time where the DAG allows /// it, then export the physical DAG. pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(root, &physical_properties_at_ingestion(root)) - .expect("physical ASAP DAG export failed") + compile_physical_asap_dag( + root, + &apply_physical_properties( + root, + &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( + std::slice::from_ref(root), + ), + ) + .expect("applying physical properties failed"), + ) + .expect("physical ASAP DAG export failed") } } diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index 3ce280a00..f269eebc8 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -9,9 +9,7 @@ use std::rc::Rc; -use asap_integration_tests::physical_export::{ - default_physical_properties, physical_asap_dag, physical_properties_at_ingestion, -}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::explicit_local_candidates; use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; use asap_types::ir::operator::operator_properties::{Reduction, Source}; @@ -200,7 +198,13 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); // Timing is not stored on the plan: time it with the outer summary // maintained (which also validates every edge) and inspect it. - let physical_properties = physical_properties_at_ingestion(&root); + let physical_properties = asap_types::ir::apply_physical_properties( + &root, + &asap_types::ir::PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( + std::slice::from_ref(&root), + ), + ) + .unwrap(); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { panic!("expected KLL evaluation, got {:?}", root.operator); }; @@ -258,9 +262,13 @@ fn max_and_avg_over_quantile_compose_at_query_time() { "the composed plan's schema is the pre-ASAP target's own" ); assert_eq!( - default_physical_properties(&composed) - .get(&composed) - .map(|properties| properties.timing()), + asap_types::ir::apply_physical_properties( + &composed, + &asap_types::ir::PhysicalPropertiesAssignment::default() + ) + .unwrap() + .get(&composed) + .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), "{intent:?}: the exact fold runs at query time" ); @@ -280,9 +288,13 @@ fn identity_and_genuine_multi_row_folds_both_compose() { let root = agg(vec![0], AggIntent::Max { col: None }, inner); let composed = composed(&root); assert_eq!( - default_physical_properties(&composed) - .get(&composed) - .map(|properties| properties.timing()), + asap_types::ir::apply_physical_properties( + &composed, + &asap_types::ir::PhysicalPropertiesAssignment::default() + ) + .unwrap() + .get(&composed) + .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), "{label}" ); diff --git a/crates/integration-tests/tests/physical_properties_e2e.rs b/crates/integration-tests/tests/physical_properties_e2e.rs index 636b8e6eb..d3c2da88a 100644 --- a/crates/integration-tests/tests/physical_properties_e2e.rs +++ b/crates/integration-tests/tests/physical_properties_e2e.rs @@ -257,7 +257,11 @@ fn mixed_placement_answers_like_query_time() { let mut mixed = PhysicalPropertiesAssignment::default(); mixed.set( &sketches(&root)[0], - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let mut answers = Vec::new(); for assignment in [PhysicalPropertiesAssignment::default(), mixed] { diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index b15a95974..9063884ac 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -10,9 +10,7 @@ use asap_types::stages::HowToComputeAggregateMeasure; use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::{ - at_ingestion_physical_asap_dag, physical_asap_dag, physical_properties_at_ingestion, -}; +use asap_integration_tests::physical_export::{at_ingestion_physical_asap_dag, physical_asap_dag}; use asap_integration_tests::planning::{ logical_candidates, logical_candidates_ignoring_guarantees, }; @@ -642,7 +640,13 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { .expect("a KLL over the finalized Sum accumulator"); // Timings are not stored on the plan: time it with its outer summary // maintained and read each node's timing. - let physical_properties = physical_properties_at_ingestion(&plan); + let physical_properties = asap_types::ir::apply_physical_properties( + &plan, + &asap_types::ir::PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( + std::slice::from_ref(&plan), + ), + ) + .unwrap(); let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = plan.asap() else { panic!("expected selected quantile summary"); }; diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs index 88a846b1c..d17270b45 100644 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ b/crates/integration-tests/tests/tumbling_window_merge.rs @@ -3,6 +3,9 @@ //! written by hand, and the planner's window-composition rule's //! `SummaryTumblingWindowChain`, expanded into the tumbling windows the executor runs. use crate::physical_common; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::collections::BTreeMap; use std::rc::Rc; @@ -281,7 +284,11 @@ fn execute_retained( for tumbling_window in tumbling_windows { assignment.set( tumbling_window, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } execute_retained_as(root, &assignment, samples) @@ -302,7 +309,11 @@ fn execute_maintained_chain( let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &chain, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(planned, &assignment).unwrap(); let expansion = expand_tumbling_window_chains_in_workload( @@ -319,7 +330,13 @@ fn execute_maintained_chain( { assert_eq!( expansion.physical_properties.get(tumbling_window), - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory) + ) + .expect("a valid timing and lifetime") + ) ); } execute_retained_as(expanded, &expansion.physical_properties, samples) diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs index d436675c6..ea980e080 100644 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ b/crates/physical-asap-optimization/src/catalog/configurations.rs @@ -6,6 +6,7 @@ //! Stage 2 validation still checks operator legality and upstream timing. //! Potential shared non-summary nodes also receive query-time `OneEvaluation`; Stage 2 //! checks highest-shared eligibility on the selected graph, not all possible consumers. +use asap_types::physical::Medium; use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; use std::collections::HashMap; @@ -39,7 +40,12 @@ pub fn may_keep_across_evaluations_per_sole_reader_query( data: &DataWorkload, query_time_results_retained: bool, ) -> Vec { - if properties != PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME + if properties + != PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime") || !matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) || !query_time_results_retained { @@ -155,21 +161,39 @@ pub fn enumerate_candidate_configurations( // whole each evaluation; // - any ingestion-time state for `SummaryEstimate`, // `EvaluatePopulation` and `MaintainPopulation` (`forced_query`). - let mut alternatives = vec![PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME]; + let mut alternatives = + vec![ + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime"), + ]; if summary { - alternatives.push(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION); + alternatives.push( + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ); if queries.iter().any(|&query| { summary_properties_allowed( node, &[query], - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), entries, data, query_time_results_retained, ) }) { alternatives.push( - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) @@ -177,7 +201,11 @@ pub fn enumerate_candidate_configurations( summary_properties_allowed( node, &[query], - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), entries, data, query_time_results_retained, @@ -185,16 +213,29 @@ pub fn enumerate_candidate_configurations( }) { alternatives.push( - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } } else if !forced_query { - alternatives.push(PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); + alternatives.push( + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime"), + ); } // Potential reach only generates the choice. Selected-reader cardinality // and Stage 2's highest-shared rule determine actual eligibility. if !summary && queries.len() >= 2 { - alternatives.push(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION); + alternatives.push( + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ); } // Configurations the deployment cannot run are not generated; each is // recorded as pruned. @@ -244,7 +285,13 @@ pub fn enumerate_candidate_configurations( query_time_results_retained, ), }); - if properties == PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION { + if properties + == PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime") + { if !summary { reader_constraints.push(ReaderConstraint::HighestSharedOneEvaluation { candidate: id, @@ -321,7 +368,7 @@ pub fn enumerate_candidate_configurations( .into_iter() .enumerate() { - let requirement = PhysicalRequirement::new(timing, Evaluation::Same) + let requirement = PhysicalRequirement::new(timing, Evaluation::Same, None) .expect("a same-evaluation read is valid at either timing"); let mut compatible = Vec::new(); for &producer in configurations { @@ -417,7 +464,7 @@ fn summary_properties_allowed( (ExecutionTiming::QueryTime, Lifetime::None) | (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => true, // Runs inside the ingestion refresh of the state that reads it, so - // the reader must allow ingestion, as for a build materialized there. + // the consumer's query must allow ingestion, as for a build materialized there. (ExecutionTiming::IngestionTime, Lifetime::None) => eligibility.at_ingestion.is_ok(), (ExecutionTiming::IngestionTime, Lifetime::AcrossEvaluations(_)) => { eligibility.at_ingestion.is_ok() diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index eaef1e232..fd089097e 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -3,6 +3,8 @@ //! No fallback modes are chosen. Physical rewrites or timing splits that change //! candidate identity are rejected until represented by physical enumeration. //! This validates execution legality, not cost estimates or accuracy targets. +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; use asap_types::physical::PhysicalProperties; use std::collections::HashMap; use std::rc::Rc; @@ -61,11 +63,14 @@ pub fn realize_selected_candidates( &mut |logical, _, options| { let node = node_by_pointer.get(&logical).copied(); // A node outside the selection is not materialized. - let requested = - node.map_or(PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME, |node| { + let requested = node.map_or( + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime"), + |node| { seen[node] = true; assignments[node] - }); + }, + ); // The option with the requested properties; option `None` (no // entry: not materialized, placed by the timing pass) for a // requested lifetime `None`, at either timing. The timing pass's diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 74a69b705..3c14c9ba5 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -119,6 +119,8 @@ pub use catalog::{ ErrorGeneratingPhysicalASAPMemoCandidates, Stage2AssignmentRealizer, }; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::collections::{HashMap, HashSet}; use std::rc::Rc; @@ -362,17 +364,29 @@ impl MaterializedOutputCandidate { let mut options = vec![None]; if self.at_ingestion.is_ok() { options.push(Some( - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), )); } if self.readers.len() > 1 { options.push(Some( - PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION, + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), )); } if self.may_keep_across_evaluations { options.push(Some( - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), )); } options @@ -699,7 +713,11 @@ pub(crate) fn find_materialized_output_candidates( for &v in below[c].iter().chain([&c]) { trial.set( &nodes[v], - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } apply_physical_properties(node, &trial) @@ -1800,7 +1818,11 @@ mod tests { let properties = |node: &OperatorNode| b1.physical_properties.get(node).unwrap(); assert_eq!( properties(chain), - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory) + ) + .expect("a valid timing and lifetime") ); for below in OperatorNode::reachable(chain) { assert_eq!(properties(&below).timing(), ExecutionTiming::IngestionTime); @@ -2413,7 +2435,13 @@ mod tests { .collect(); let options = vec![ None, - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION), + Some( + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ), ]; assert_eq!( found, diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs index c4f1d4349..785f5b1dc 100644 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ b/crates/physical-asap-optimization/src/split_by_timing.rs @@ -261,7 +261,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &summary, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); let physical_properties = apply_physical_properties(&split[0], &assignment).unwrap(); @@ -272,7 +276,13 @@ mod tests { )); assert_eq!( physical_properties.get(&summary), - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory) + ) + .expect("a valid timing and lifetime") + ) ); } @@ -295,7 +305,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &summary, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let as_roots = |roots: &[Rc]| -> Vec { roots.iter().cloned().map(QueryRoot::Operator).collect() diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs index af1a3b328..1216f735c 100644 --- a/crates/plan-selection/src/groups.rs +++ b/crates/plan-selection/src/groups.rs @@ -1095,6 +1095,28 @@ fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { fn window_read_through( path: &[&PhysicalASAPDAGNode], ) -> Result { + // A chain materialized across evaluations at query time builds only its + // newest pane per evaluation, so its scan reads one pane. + let materialized_across_evaluations_at_query_time = |node: &PhysicalASAPDAGNode| { + use ExecutionTiming::{IngestionTime, QueryTime}; + let properties = PhysicalProperties::new(node.output_state.timing, node.lifetime) + .map_err(|reason| { + asap_cost::analytical::AnalyticalCostError::InvalidPhysicalDag( + asap_types::ir::physical_export::PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: node.id, + reason, + }, + ) + })?; + Ok(match (properties.timing(), properties.lifetime()) { + (QueryTime, Lifetime::AcrossEvaluations(_)) => true, + (QueryTime, Lifetime::None | Lifetime::OneEvaluation(_)) + | (IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_)) => false, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by PhysicalProperties::new") + } + }) + }; let mut end_offset_ms: i64 = 0; for node in path { let width_ms: u128 = match &node.payload { @@ -1106,7 +1128,7 @@ fn window_read_through( Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - }) if asap_cost::is_query_time_materialized_tumbling_window_chain(node)? => { + }) if materialized_across_evaluations_at_query_time(node)? => { u128::from(*tumbling_window_width_ms) } Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs index 8ecd5be77..90473d339 100644 --- a/crates/plan-selection/src/groups/tests.rs +++ b/crates/plan-selection/src/groups/tests.rs @@ -220,7 +220,11 @@ fn dominance_needs_every_condition() { ( "lifetime", DominanceKey { - properties: PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION, + properties: PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ..key.clone() }, ), @@ -489,7 +493,11 @@ fn shared_middle() -> PhysicalASAPMemoCandidates { let candidates = vec![ candidate(0, &[], false), PhysicalCandidate { - properties: PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION, + properties: PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ..candidate(1, &[&[0]], false) }, candidate(2, &[&[1]], true), @@ -1124,7 +1132,21 @@ fn run_kind_over_every_valid_state() { ), ]); } - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let (_, kind) = expected .iter() .find(|(state, _)| *state == properties) diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 5ba637e8f..63262babe 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -3544,12 +3544,26 @@ mod tests { } /// Materialized in memory at ingestion time until a query reads it. - const AT_INGESTION: Option = - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION); + fn at_ingestion() -> Option { + Some( + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ) + } /// Materialized in memory at query time for one evaluation. - const AT_QUERY: Option = - Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION); + fn at_query() -> Option { + Some( + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::OneEvaluation(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ) + } fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id @@ -3586,7 +3600,7 @@ mod tests { /// node scales with that rate. #[test] fn ingestion_time_node_is_priced_per_second_of_arriving_rows() { - let dag = kll_dag(AT_INGESTION); + let dag = kll_dag(at_ingestion()); let (fast, slow) = ( priced(&dag, &[repeating(1_000)]), priced(&dag, &[repeating(10_000)]), @@ -3618,7 +3632,7 @@ mod tests { let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); price(dag, &recurrences(1), &data(), &settings).unwrap() }; - for materialized in [AT_INGESTION, AT_QUERY, None] { + for materialized in [at_ingestion(), at_query(), None] { let dag = kll_dag(materialized); let build = node_of(&dag, is_build); let (with, without) = (cost(&dag, 1.25e-7), cost(&dag, 0.0)); @@ -3631,7 +3645,7 @@ mod tests { // each sees its series' 12 samples: the 120 000 rows of its // source's 1-min window (data()'s 10 000 rows per second over its // five sources); at ingestion time its samples are unknown. - let samples = if materialized == AT_INGESTION { + let samples = if materialized == at_ingestion() { StateInputRows::UNKNOWN } else { StateInputRows { @@ -3640,7 +3654,7 @@ mod tests { } }; let state = 10_000.0 * summary_shape(family, samples).1 as f64; - let byte_seconds = if materialized == AT_INGESTION { + let byte_seconds = if materialized == at_ingestion() { 2.0 * state } else { let evaluation_s = with.usage.latency_per_query[0].total_ms() / 1_000.0; @@ -3699,7 +3713,7 @@ mod tests { (query.to_string(), dag) }) .collect(); - for materialized in [AT_INGESTION, AT_QUERY, None] { + for materialized in [at_ingestion(), at_query(), None] { dags.push((format!("{materialized:?}"), kll_dag(materialized))); } for (name, dag) in &dags { @@ -4664,7 +4678,7 @@ mod tests { Lifetime::AcrossEvaluations(Medium::Disk), ) .ok(), - AT_INGESTION, + at_ingestion(), ), ( PhysicalProperties::new( @@ -4672,7 +4686,7 @@ mod tests { Lifetime::OneEvaluation(Medium::Disk), ) .ok(), - AT_QUERY, + at_query(), ), ] { let (disk_dag, memory_dag) = (kll_dag(on_disk), kll_dag(in_memory)); @@ -5053,7 +5067,7 @@ mod tests { /// materialized runs at query time). fn kll_dag_with(properties: PhysicalProperties) -> PhysicalASAPDAG { match (properties.timing(), properties.lifetime()) { - (ExecutionTiming::IngestionTime, Lifetime::None) => kll_dag(AT_INGESTION), + (ExecutionTiming::IngestionTime, Lifetime::None) => kll_dag(at_ingestion()), _ => kll_dag(Some(properties)), } } @@ -5065,12 +5079,27 @@ mod tests { fn cost_over_every_valid_state() { let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); let not_materialized = price(&kll_dag(None), &recurrences(1), &data(), &settings).unwrap(); - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let dag = kll_dag_with(properties); assert!( - dag.nodes - .iter() - .any(|node| node.physical_properties() == Ok(properties)), + dag.nodes.iter().any(|node| PhysicalProperties::new( + node.output_state.timing, + node.lifetime + ) == Ok(properties)), "{properties:?}" ); let cost = price(&dag, &recurrences(1), &data(), &settings) @@ -5099,14 +5128,31 @@ mod tests { use asap_cost::execution_demand::computations; use ExecutionTiming::{IngestionTime, QueryTime}; let per_minute = 1.0 / 60.0; - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let dag = kll_dag_with(properties); let times = computations(&dag, &[repeating(MINUTE)], &[], 3600.0, None) .unwrap_or_else(|error| panic!("{properties:?}: {error}")); let node = dag .nodes .iter() - .find(|node| node.physical_properties() == Ok(properties)) + .find(|node| { + PhysicalProperties::new(node.output_state.timing, node.lifetime) + == Ok(properties) + }) .unwrap_or_else(|| panic!("{properties:?}")); let (_, rate) = times[&node.id]; let expected = match (properties.timing(), properties.lifetime()) { @@ -5137,7 +5183,21 @@ mod tests { let rows_per_s = 1e9; let b1 = tumbling_window_candidates(rows_per_s).remove(1); let chain = node_of(&b1.dag, is_chain); - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let mut dag = b1.dag.clone(); let node = dag.nodes.iter_mut().find(|n| n.id == chain).unwrap(); node.output_state.timing = properties.timing(); diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs index c8e93883a..e106299f6 100644 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ b/crates/plan-selection/src/milp/cost_terms.rs @@ -707,7 +707,7 @@ fn pane_retention_terms( indicators.push(direct.into()); } for &reader in milp - .materialized_readers + .materialized_consumers .get(&chain) .cloned() .unwrap_or_default() diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 292453074..75fc80919 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -249,7 +249,7 @@ pub(crate) struct SelectionMilp<'a> { pub(crate) reach: HashMap<(usize, PhysicalCandidateId), Variable>, /// Per materialized candidate, the materialized candidates whose contexts may /// read it. - pub(crate) materialized_readers: BTreeMap>, + pub(crate) materialized_consumers: BTreeMap>, /// `L[q]`. pub(crate) latency: Vec, /// An upper bound of each `L[q]` (its bound, or the most it can be), in @@ -498,7 +498,7 @@ pub(crate) fn build_selection_milp<'a>( readers: Vec::new(), selected: BTreeMap::new(), reach: HashMap::new(), - materialized_readers: BTreeMap::new(), + materialized_consumers: BTreeMap::new(), latency: Vec::new(), latency_max: Vec::new(), ms_per_latency_unit: 1.0, @@ -628,7 +628,7 @@ fn choice_variables(milp: &mut SelectionMilp<'_>) { if let ContextKind::Materialized(reader) = data.kind { for &option in &data.options { if inputs.kind[option.0] == RunKind::Materialized { - milp.materialized_readers + milp.materialized_consumers .entry(option) .or_default() .push(reader); @@ -1030,7 +1030,7 @@ pub(crate) fn strongly_connected_components(count: usize, edges: &[(usize, usize /// candidate its context may read (readers first), so `a[q, m]` exists /// before the reach rows of what `m` reads. A cycle among contexts is /// broken at the first candidate met. -pub(crate) fn materialized_readers_first(milp: &SelectionMilp<'_>) -> Vec { +pub(crate) fn materialized_consumers_first(milp: &SelectionMilp<'_>) -> Vec { let inputs = milp.inputs; let mut order = Vec::new(); let mut state: BTreeMap = BTreeMap::new(); @@ -1068,7 +1068,7 @@ pub(crate) fn materialized_readers_first(milp: &SelectionMilp<'_>) -> Vec) { let inputs = milp.inputs; milp.builder.scope = Scope::MaterializedContexts; - let order = materialized_readers_first(milp); + let order = materialized_consumers_first(milp); // Which queries may depend on each materialized candidate: read in their // context, or in the context of one they may depend on. let mut may_reach: BTreeMap> = BTreeMap::new(); @@ -1105,7 +1105,7 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { .row("6 reach from below", constraint!(reach >= direct)); } for &reader in milp - .materialized_readers + .materialized_consumers .get(&materialized) .into_iter() .flatten() @@ -1135,7 +1135,7 @@ fn reach_rows(milp: &mut SelectionMilp<'_>) { sources += direct; } for &reader in milp - .materialized_readers + .materialized_consumers .get(&materialized) .cloned() .unwrap_or_default() @@ -1200,7 +1200,7 @@ fn materialized_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet) -> BTreeSet) { } } for &reader in milp - .materialized_readers + .materialized_consumers .get(&candidate) .cloned() .unwrap_or_default() diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs index c59c2c3b4..930c45702 100644 --- a/crates/plan-selection/src/milp/model/tests.rs +++ b/crates/plan-selection/src/milp/model/tests.rs @@ -838,7 +838,7 @@ fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { .collect(); let m = *constrained .first() - .expect("a two-reader materialized result"); + .expect("a materialized result with two consumers"); let selected = milp.selected[&m]; milp.builder.row("test", constraint!(selected >= 1.0)); let readers: Vec = milp.readers[1][&m] diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index 6ede3b230..d178714ef 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -25,9 +25,7 @@ use std::collections::BTreeMap; use std::time::Instant; -use asap_cost::analytical::AnalyticalCostError; use asap_cost::{materialized_tumbling_window_chains, node_map, plan_cost, CostInputs}; -use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, @@ -49,18 +47,6 @@ use crate::report::{ Selection, }; -/// [`ErrorSelectingPlan::Cost`] of `candidate`, from `plan_cost`'s error at -/// a node. -pub(crate) fn cost_error( - candidate: &str, -) -> impl Fn((PhysicalASAPNodeId, AnalyticalCostError)) -> ErrorSelectingPlan + '_ { - move |(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.to_owned(), - node, - error, - } -} - /// The largest relative difference between the objective and `plan_cost` /// that counts as equal. const COST_EQUALITY_TOLERANCE: f64 = 1e-6; @@ -142,14 +128,24 @@ pub fn select_plan( let (cost, _) = plan_cost( &plan.dag, &node_map(&plan.dag), - &materialized_tumbling_window_chains(&plan.dag).map_err(cost_error(&plan.id))?, + &materialized_tumbling_window_chains(&plan.dag).map_err(|(node, error)| { + ErrorSelectingPlan::Cost { + candidate: plan.id.clone(), + node, + error, + } + })?, &recurrence, &ranges, data, cost_inputs, formula_scale, ) - .map_err(cost_error(&plan.id))?; + .map_err(|(node, error)| ErrorSelectingPlan::Cost { + candidate: plan.id.clone(), + node, + error, + })?; let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) })? - solution.request_cost_per_second; diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs index 7cc2daf94..afe1d8a59 100644 --- a/crates/plan-selection/src/selector.rs +++ b/crates/plan-selection/src/selector.rs @@ -16,7 +16,6 @@ use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; use crate::report::{ AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, }; -use crate::select::cost_error; use crate::SelectionModels; use asap_cost::analytical::AnalyticalCostError; use asap_cost::{ @@ -182,8 +181,14 @@ impl<'a> PlanSelector<'a> { return Ok(CalibrationFit::default()); } let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_tumbling_window_chains(&candidate.dag) - .map_err(cost_error(&candidate.id))?; + let materialized_chains = + materialized_tumbling_window_chains(&candidate.dag).map_err(|(node, error)| { + ErrorSelectingPlan::Cost { + candidate: candidate.id.clone(), + node, + error, + } + })?; let (_, fit) = plan_cost( &candidate.dag, &nodes, @@ -194,7 +199,11 @@ impl<'a> PlanSelector<'a> { self.models.cost_inputs, 1.0, ) - .map_err(cost_error(&candidate.id))?; + .map_err(|(node, error)| ErrorSelectingPlan::Cost { + candidate: candidate.id.clone(), + node, + error, + })?; Ok(fit) } @@ -229,8 +238,14 @@ impl<'a> PlanSelector<'a> { } }; let nodes = node_map(&candidate.dag); - let materialized_chains = materialized_tumbling_window_chains(&candidate.dag) - .map_err(cost_error(&candidate.id))?; + let materialized_chains = + materialized_tumbling_window_chains(&candidate.dag).map_err(|(node, error)| { + ErrorSelectingPlan::Cost { + candidate: candidate.id.clone(), + node, + error, + } + })?; let settings = self.models.cost_inputs; let (cost, _) = plan_cost( &candidate.dag, @@ -242,7 +257,11 @@ impl<'a> PlanSelector<'a> { settings, self.formula_scale, ) - .map_err(cost_error(&candidate.id))?; + .map_err(|(node, error)| ErrorSelectingPlan::Cost { + candidate: candidate.id.clone(), + node, + error, + })?; let violation = latency_violation(&cost, self.entries).or_else(|| match &settings.cost_model { CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits) diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index fadd2db0b..a9b9e9414 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -802,6 +802,8 @@ mod tests { use super::*; use asap_types::ir::schema::{SketchAlgorithm, StatisticKind, SupportedSummary}; use asap_types::ir::{NonASAPOp, Operator}; + use asap_types::physical::Lifetime; + use asap_types::physical::Medium; use asap_types::physical::PhysicalProperties; thread_local! { @@ -1001,7 +1003,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &merged, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&merged, &assignment).unwrap(); let dag = compile_physical_asap_dag(&merged, &physical_properties).unwrap(); @@ -1039,7 +1045,11 @@ mod tests { for node in tumbling_windows.iter().chain([&merge]) { assignment.set( node, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } let physical_properties = apply_physical_properties(&merge, &assignment).unwrap(); diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 381f52970..30e13dae9 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -401,10 +401,6 @@ impl OptimizationPass for Stage1ToStage3Pass { roots, physical_properties, } = run.selected_plan.expect("a selection has a plan"); - debug_assert!( - physical_properties.lists_every_node_of(&roots), - "the selected plan's physical properties list every node of its roots" - ); let mut output = PlanOutput::new(Vec::new()); for (entry_index, root) in roots.iter().enumerate() { match root { diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index af2bc3cb9..2cdde527c 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -4,6 +4,9 @@ //! the compiler checks that no internal is used (CI runs //! `cargo check -p asap-planner --tests --locked` on its own for that). +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::sync::Arc; @@ -1577,14 +1580,24 @@ impl OptimizationPass for StoredAsForced { /// Every `SummaryAgg` built at ingestion time. fn summaries_at_ingestion(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })) - .then(|| PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) + matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })).then(|| { + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime") + }) } /// Every `SummaryMerge` built at query time and materialized across evaluations. fn merges_materialized(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })) - .then(|| PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME) + matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })).then(|| { + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime") + }) } /// Exact PromQL `queries`, every minute, over 6 ingesting series of `m` diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index c3b0ae9e3..eee0787c6 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -23,7 +23,7 @@ use super::query::QueryRoot; use crate::ir::properties::timing::PhysicalPropertiesAssignment; use crate::ir::properties::ExecutionTiming; use crate::ir::SchemaDerivationError; -use crate::physical::PhysicalProperties; +use crate::physical::{Lifetime, PhysicalProperties}; fn invalid(message: &str) -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(message.into()) @@ -365,7 +365,11 @@ fn not_materialized_below( ) { for child in node.children() { if properties.get(child).is_none() { - properties.set(child, PhysicalProperties::not_materialized_at(timing)); + properties.set( + child, + PhysicalProperties::new(timing, Lifetime::None) + .expect("a valid timing and lifetime"), + ); not_materialized_below(child, timing, properties); } } diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 8677e9a2f..61ae190a0 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -92,14 +92,6 @@ pub struct PhysicalASAPDAGNode { pub retention_ms: Option, } -impl PhysicalASAPDAGNode { - /// Its timing (`output_state.timing`) and lifetime, unless they cannot - /// go together. - pub fn physical_properties(&self) -> Result { - PhysicalProperties::new(self.output_state.timing, self.lifetime) - } -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PhysicalASAPDAGEdge { @@ -619,14 +611,17 @@ impl PhysicalASAPDAG { // A node's timing and lifetime must go together; a summary // build that is not materialized runs at query time. Any other node that // is not materialized runs when its consumer runs. - let properties = node.physical_properties().map_err(|reason| { - PhysicalASAPDAGValidationError::InvalidPhysicalProperties { - node: node.id, - reason, - } - })?; + let properties = PhysicalProperties::new(node.output_state.timing, node.lifetime) + .map_err( + |reason| PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: node.id, + reason, + }, + )?; if is_materializable_state_output(&node.payload) - && properties == PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION + && properties + == PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime") { return Err( PhysicalASAPDAGValidationError::UnmaterializedSummaryBuildNotAtQueryTime { diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs index d175a8700..3529faf93 100644 --- a/crates/types/src/ir/properties/timing.rs +++ b/crates/types/src/ir/properties/timing.rs @@ -63,9 +63,9 @@ use crate::ir::properties::execution_data_state::{ DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; use crate::ir::schema::{DataType, FieldDataType, Schema}; -use crate::physical::{satisfies_physical, Evaluation, PhysicalProperties, PhysicalRequirement}; -#[cfg(test)] -use crate::physical::{Lifetime, Medium}; +use crate::physical::{ + satisfies_physical, Evaluation, Lifetime, Medium, PhysicalProperties, PhysicalRequirement, +}; /// The physical properties of operator nodes, by node identity (#509 /// §Materialization: "Materialization decides, for each sub-DAG, whether its @@ -139,30 +139,6 @@ impl PhysicalPropertiesAssignment { .map(|(_, properties)| *properties) } - /// Whether the assignment lists every node reachable from `roots`, as a - /// completed assignment ([`apply_physical_properties_to_roots`]) does. - pub fn lists_every_node_of(&self, roots: &[QueryRoot]) -> bool { - fn visit( - node: &OperatorNode, - assignment: &PhysicalPropertiesAssignment, - visited: &mut std::collections::HashSet<*const OperatorNode>, - ) -> bool { - if !visited.insert(node as *const OperatorNode) { - return true; - } - assignment.get(node).is_some() - && node - .children() - .iter() - .all(|child| visit(child, assignment, visited)) - } - let mut visited = std::collections::HashSet::new(); - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .all(|node| visit(node, self, &mut visited)) - } - /// Every summary build and `MaintainPopulation` reachable from `roots` /// materialized in memory at ingestion time where the DAG allows it: not under a /// filter, and with nothing below it that can only run at query time. @@ -188,7 +164,11 @@ impl PhysicalPropertiesAssignment { { assignment.set( node, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } } @@ -447,9 +427,10 @@ fn placement( // A summary build that is not materialized runs at query time. Ok( if runs_at_query_time_only(node) || is_materializable_state_output(&node.operator) { - PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime") } else { - PhysicalProperties::not_materialized_at(consumer) + PhysicalProperties::new(consumer, Lifetime::None).expect("a valid timing and lifetime") }, ) } @@ -868,10 +849,10 @@ fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { } } -/// r^phys of a reader at `timing` that reads its input's output in the same +/// r^phys of a consumer at `timing` that reads its input's output in the same /// evaluation (or ingestion work), as every edge of the timing pass does. fn read_in_the_same_evaluation(timing: ExecutionTiming) -> PhysicalRequirement { - PhysicalRequirement::new(timing, Evaluation::Same) + PhysicalRequirement::new(timing, Evaluation::Same, None) .expect("a same-evaluation read is valid at either timing") } @@ -976,7 +957,11 @@ mod tests { fn fixed_at_ingestion(nodes: &[&Rc]) -> PhysicalPropertiesAssignment { let mut assignment = PhysicalPropertiesAssignment::default(); for node in nodes { - assignment.set(node, PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); + assignment.set( + node, + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime"), + ); } assignment } @@ -992,7 +977,11 @@ mod tests { std::slice::from_ref(root), ); for node in fixed { - assignment.set(node, PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); + assignment.set( + node, + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime"), + ); } apply_physical_properties(root, &assignment) } @@ -1080,7 +1069,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &outer, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); assert_eq!( apply_physical_properties(&estimate(outer), &assignment).err(), @@ -1090,12 +1083,20 @@ mod tests { ); assignment.set( &inner, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let outer = agg(Rc::clone(&inner), kll()); assignment.set( &outer, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&estimate(Rc::clone(&outer)), &assignment).unwrap(); @@ -1251,12 +1252,23 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &node, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), + ); + assignment.set( + &node, + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime"), ); - assignment.set(&node, PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME); assert_eq!( assignment.get(&node), - Some(PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME) + Some( + PhysicalProperties::new(ExecutionTiming::QueryTime, Lifetime::None) + .expect("a valid timing and lifetime") + ) ); } @@ -1270,7 +1282,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &tumbling_windows, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); assert_eq!( @@ -1294,7 +1310,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &over_evaluation, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); assert_eq!( apply_physical_properties(&over_evaluation, &assignment).err(), @@ -1341,7 +1361,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &filtered, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); assert_eq!( apply_physical_properties(&estimate(Rc::clone(&filtered)), &assignment).err(), @@ -1409,7 +1433,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &agg(scan(), kll()), - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let text = format!("{assignment:?}"); assert!(text.contains("SummaryAgg@"), "{text}"); @@ -1436,7 +1464,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &rows, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&summary, &assignment).unwrap(); assert_eq!( @@ -1468,7 +1500,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &state, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&summary, &assignment).unwrap(); assert_eq!( @@ -1490,7 +1526,11 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &root, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); assert_eq!( @@ -1507,11 +1547,19 @@ mod tests { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &state, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); assignment.set( &root, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); assert_eq!( diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index 0a417ff4a..aaf67372a 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,7 +1,7 @@ //! #509 Stage 2 decision data: the physical properties of each candidate. //! //! - [`properties`] — when a candidate runs and how long its output lives -//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]), and what a reader +//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]), and what a consumer //! requires of them ([`PhysicalRequirement`], [`satisfies_physical`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). @@ -9,7 +9,7 @@ pub mod properties; pub mod workload_plan; pub use properties::{ - satisfies_physical, Evaluation, InvalidPhysicalProperties, InvalidPhysicalRequirement, - Lifetime, Medium, PhysicalProperties, PhysicalRequirement, + satisfies_physical, Evaluation, EvaluationSchedule, InvalidPhysicalProperties, Lifetime, + Medium, PhysicalProperties, PhysicalRequirement, }; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 8260bf6e6..7917c6fe7 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -3,8 +3,8 @@ //! //! | timing | lifetime | how it runs | //! |---|---|---| -//! | query | `None` | in each evaluation that reads it; discarded after its reader | -//! | query | `OneEvaluation` | once per evaluation; its readers in that evaluation share it | +//! | query | `None` | in each evaluation that reads it; discarded after its consumer runs | +//! | query | `OneEvaluation` | once per evaluation; its consumers in that evaluation share it | //! | query | `AcrossEvaluations` | at query time, materialized for later evaluations (a tumbling-window chain that builds its newest window each evaluation) | //! | ingestion | `None` | as data arrives, inside the ingestion work that reads it | //! | ingestion | `AcrossEvaluations` | as data arrives; materialized until a query reads it | @@ -24,20 +24,24 @@ pub enum Medium { Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] pub enum Lifetime { - /// Discarded after its reader's execution completes. + /// Discarded after its consumer's execution completes. #[default] None, - /// Materialized until every reader within one evaluation finishes. + /// Materialized until every consumer within one evaluation finishes. OneEvaluation(Medium), /// Materialized across evaluations, so later evaluations reuse it. AcrossEvaluations(Medium), } -/// A timing and a lifetime that cannot go together. +/// A physical property or physical requirement whose dimensions cannot go +/// together: bad input to [`PhysicalProperties::new`] or +/// [`PhysicalRequirement::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InvalidPhysicalProperties { #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] OneEvaluationAtIngestion, + #[error("an ingestion-time consumer cannot read a later evaluation's output: there is no evaluation at ingestion time")] + LaterEvaluationAtIngestion, } /// When a candidate runs and how long its output lives: one of the valid @@ -49,33 +53,6 @@ pub struct PhysicalProperties { } impl PhysicalProperties { - /// Query time, not materialized: runs in each reader's execution. - pub const NOT_MATERIALIZED_AT_QUERY_TIME: Self = Self { - timing: ExecutionTiming::QueryTime, - lifetime: Lifetime::None, - }; - /// Ingestion time, not materialized: runs inside the ingestion work that reads it. - pub const NOT_MATERIALIZED_AT_INGESTION: Self = Self { - timing: ExecutionTiming::IngestionTime, - lifetime: Lifetime::None, - }; - /// Query time, materialized in memory until the evaluation's readers finish. - pub const MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION: Self = Self { - timing: ExecutionTiming::QueryTime, - lifetime: Lifetime::OneEvaluation(Medium::Memory), - }; - /// Ingestion time, materialized in memory until a query reads it. - pub const MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION: Self = Self { - timing: ExecutionTiming::IngestionTime, - lifetime: Lifetime::AcrossEvaluations(Medium::Memory), - }; - /// Query time, materialized in memory across evaluations (a tumbling-window chain - /// that builds its newest window at each evaluation). - pub const MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME: Self = Self { - timing: ExecutionTiming::QueryTime, - lifetime: Lifetime::AcrossEvaluations(Medium::Memory), - }; - /// The properties `timing` and `lifetime`, unless they cannot go together. pub fn new( timing: ExecutionTiming, @@ -94,14 +71,6 @@ impl PhysicalProperties { } } - /// `(timing, None)`: not materialized, at `timing`. - pub fn not_materialized_at(timing: ExecutionTiming) -> Self { - Self { - timing, - lifetime: Lifetime::None, - } - } - pub fn timing(self) -> ExecutionTiming { self.timing } @@ -110,7 +79,7 @@ impl PhysicalProperties { self.lifetime } - /// Whether the output outlives its reader (lifetime `OneEvaluation` or + /// Whether the output outlives its consumer (lifetime `OneEvaluation` or /// `AcrossEvaluations`): materialized(v) ⇔ ℓ_v ≠ None. pub fn is_materialized(self) -> bool { use ExecutionTiming::{IngestionTime, QueryTime}; @@ -123,80 +92,62 @@ impl PhysicalProperties { } } } - - /// The 8 valid states, as literal `(timing, lifetime)` pairs through - /// [`Self::new`], for table tests over every state. - #[cfg(any(test, feature = "test-support"))] - pub fn every_valid() -> [Self; 8] { - use ExecutionTiming::{IngestionTime, QueryTime}; - use Medium::{Disk, Memory}; - [ - (QueryTime, Lifetime::None), - (IngestionTime, Lifetime::None), - (QueryTime, Lifetime::OneEvaluation(Memory)), - (QueryTime, Lifetime::OneEvaluation(Disk)), - (QueryTime, Lifetime::AcrossEvaluations(Memory)), - (QueryTime, Lifetime::AcrossEvaluations(Disk)), - (IngestionTime, Lifetime::AcrossEvaluations(Memory)), - (IngestionTime, Lifetime::AcrossEvaluations(Disk)), - ] - .map(|(timing, lifetime)| Self::new(timing, lifetime).expect("a valid pair")) - } } -/// Which evaluation's output a reader reads. +/// Which evaluation's output a consumer reads. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Evaluation { - /// The output of the evaluation the reader runs in (or, at ingestion + /// The output of the evaluation the consumer runs in (or, at ingestion /// time, of the ingestion work it runs in). Same, /// The output of an earlier evaluation, reused by a later one. Later, } -/// A reader timing and an evaluation that cannot go together. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum InvalidPhysicalRequirement { - #[error("an ingestion-time reader cannot read a later evaluation's output: there is no evaluation at ingestion time")] - LaterEvaluationAtIngestion, +/// The evaluation schedule (I_κ, φ_κ) of a consumer's execution context, +/// taken from its query's 𝒯_q: it evaluates at φ_κ + I_κ·ℤ. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationSchedule { + /// I_κ, in milliseconds. + pub interval_ms: u64, + /// φ_κ, in milliseconds. + pub phase_ms: u64, } -/// r^phys_{c,i}: when a consumer reads one input, and from which -/// evaluation. The reader's schedule comes with the schedule σ (#1200 PR 2). +/// r^phys_{c,i}: when a consumer reads one input, from which evaluation, and +/// on which schedule. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalRequirement { - reader_timing: ExecutionTiming, + consumer_timing: ExecutionTiming, reads_from: Evaluation, + /// `None` for a consumer whose query is not periodic. No caller states one until the + /// producers' schedule σ exists (#1200 PR 2). + consumer_schedule: Option, } impl PhysicalRequirement { - /// The requirement of a reader at `reader_timing` reading `reads_from`, - /// unless they cannot go together. + /// The requirement of a consumer at `consumer_timing` reading `reads_from` + /// on `consumer_schedule`, unless they cannot go together: an + /// ingestion-time consumer cannot read a later evaluation. pub fn new( - reader_timing: ExecutionTiming, + consumer_timing: ExecutionTiming, reads_from: Evaluation, - ) -> Result { + consumer_schedule: Option, + ) -> Result { use ExecutionTiming::{IngestionTime, QueryTime}; - match (reader_timing, reads_from) { + match (consumer_timing, reads_from) { (IngestionTime, Evaluation::Later) => { - Err(InvalidPhysicalRequirement::LaterEvaluationAtIngestion) + Err(InvalidPhysicalProperties::LaterEvaluationAtIngestion) } (QueryTime | IngestionTime, Evaluation::Same) | (QueryTime, Evaluation::Later) => { Ok(Self { - reader_timing, + consumer_timing, reads_from, + consumer_schedule, }) } } } - - pub fn reader_timing(self) -> ExecutionTiming { - self.reader_timing - } - - pub fn reads_from(self) -> Evaluation { - self.reads_from - } } /// α^phys_{u,c,i}: whether a producer with `producer`'s properties may serve @@ -211,10 +162,13 @@ pub fn satisfies_physical( requirement: &PhysicalRequirement, ) -> bool { use ExecutionTiming::{IngestionTime, QueryTime}; + // #1200's schedule table needs the producer's σ (#1200 PR 2); until then + // no consumer states a schedule. + debug_assert!(requirement.consumer_schedule.is_none()); match ( producer.timing, producer.lifetime, - requirement.reader_timing, + requirement.consumer_timing, requirement.reads_from, ) { (IngestionTime, Lifetime::OneEvaluation(_), ..) => { @@ -223,7 +177,7 @@ pub fn satisfies_physical( (_, _, IngestionTime, Evaluation::Later) => { unreachable!("rejected by PhysicalRequirement::new") } - // An ingestion-time reader reads what runs at ingestion time. + // An ingestion-time consumer reads what runs at ingestion time. ( IngestionTime, Lifetime::None | Lifetime::AcrossEvaluations(_), @@ -231,7 +185,7 @@ pub fn satisfies_physical( Evaluation::Same, ) => true, (QueryTime, _, IngestionTime, Evaluation::Same) => false, - // A query-time reader in the same evaluation. + // A query-time consumer in the same evaluation. (QueryTime, _, QueryTime, Evaluation::Same) => true, (IngestionTime, Lifetime::AcrossEvaluations(_), QueryTime, Evaluation::Same) => true, (IngestionTime, Lifetime::None, QueryTime, Evaluation::Same) => false, @@ -244,65 +198,60 @@ pub fn satisfies_physical( #[cfg(test)] mod tests { use super::*; + use ExecutionTiming::{IngestionTime, QueryTime}; - /// Of every timing x lifetime pair, exactly the 8 of `every_valid` are - /// valid, and the 2 at ingestion time for one evaluation are not. + /// Every timing x lifetime pair `new` accepts. + fn valid_pairs() -> Vec { + [QueryTime, IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + .filter_map(Result::ok) + .collect() + } + + /// Of the 10 timing x lifetime pairs, exactly the 2 at ingestion time for + /// one evaluation are invalid. #[test] fn exactly_the_ingestion_time_one_evaluation_pairs_are_invalid() { - use ExecutionTiming::{IngestionTime, QueryTime}; - let mut valid = Vec::new(); for timing in [QueryTime, IngestionTime] { - for lifetime in [ - Lifetime::None, - Lifetime::OneEvaluation(Medium::Memory), - Lifetime::OneEvaluation(Medium::Disk), - Lifetime::AcrossEvaluations(Medium::Memory), - Lifetime::AcrossEvaluations(Medium::Disk), - ] { - match PhysicalProperties::new(timing, lifetime) { - Ok(properties) => valid.push(properties), - Err(error) => { - assert_eq!(error, InvalidPhysicalProperties::OneEvaluationAtIngestion); - assert!( - timing == IngestionTime - && matches!(lifetime, Lifetime::OneEvaluation(_)) - ); - } + for medium in [Medium::Memory, Medium::Disk] { + let one_evaluation = + PhysicalProperties::new(timing, Lifetime::OneEvaluation(medium)); + match timing { + IngestionTime => assert_eq!( + one_evaluation, + Err(InvalidPhysicalProperties::OneEvaluationAtIngestion) + ), + QueryTime => assert!(one_evaluation.is_ok()), } } } - let every_valid = PhysicalProperties::every_valid(); - assert_eq!(valid.len(), every_valid.len()); - assert!(valid.iter().all(|state| every_valid.contains(state))); - } - - /// The named properties are valid pairs. - #[test] - fn named_properties_are_valid() { - for named in [ - PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME, - PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION, - PhysicalProperties::MATERIALIZED_IN_MEMORY_FOR_ONE_EVALUATION, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, - ] { - assert_eq!( - PhysicalProperties::new(named.timing(), named.lifetime()), - Ok(named) - ); - } + assert_eq!(valid_pairs().len(), 8); } /// `satisfies_physical` for every valid state and every valid requirement. #[test] fn satisfies_physical_over_every_valid_state() { - use ExecutionTiming::{IngestionTime, QueryTime}; let at = |timing, lifetime| PhysicalProperties::new(timing, lifetime).unwrap(); - let same_at_query = PhysicalRequirement::new(QueryTime, Evaluation::Same).unwrap(); - let same_at_ingestion = PhysicalRequirement::new(IngestionTime, Evaluation::Same).unwrap(); - let later_at_query = PhysicalRequirement::new(QueryTime, Evaluation::Later).unwrap(); + let requirement = + |timing, evaluation| PhysicalRequirement::new(timing, evaluation, None).unwrap(); + let same_at_query = requirement(QueryTime, Evaluation::Same); + let same_at_ingestion = requirement(IngestionTime, Evaluation::Same); + let later_at_query = requirement(QueryTime, Evaluation::Later); // (state, query time same evaluation, ingestion time, later evaluation) - let mut expected = Vec::new(); + let mut expected = vec![ + (at(QueryTime, Lifetime::None), true, false, false), + (at(IngestionTime, Lifetime::None), false, true, false), + ]; for medium in [Medium::Memory, Medium::Disk] { expected.extend([ ( @@ -325,29 +274,21 @@ mod tests { ), ]); } - expected.extend([ - (at(QueryTime, Lifetime::None), true, false, false), - (at(IngestionTime, Lifetime::None), false, true, false), - ]); - assert_eq!(expected.len(), PhysicalProperties::every_valid().len()); - for properties in PhysicalProperties::every_valid() { - let (_, query, ingestion, later) = expected - .iter() - .find(|(state, ..)| *state == properties) - .expect("every valid state has an expectation"); + assert_eq!(expected.len(), valid_pairs().len()); + for (properties, query, ingestion, later) in expected { assert_eq!( satisfies_physical(&properties, &same_at_query), - *query, + query, "{properties:?}" ); assert_eq!( satisfies_physical(&properties, &same_at_ingestion), - *ingestion, + ingestion, "{properties:?}" ); assert_eq!( satisfies_physical(&properties, &later_at_query), - *later, + later, "{properties:?}" ); } @@ -356,7 +297,7 @@ mod tests { /// `is_materialized` is lifetime ≠ None for every valid state. #[test] fn is_materialized_over_every_valid_state() { - for properties in PhysicalProperties::every_valid() { + for properties in valid_pairs() { assert_eq!( properties.is_materialized(), properties.lifetime() != Lifetime::None, @@ -365,25 +306,20 @@ mod tests { } } - /// An ingestion-time reader cannot read a later evaluation; every other + /// An ingestion-time consumer cannot read a later evaluation; every other /// requirement is valid. #[test] fn later_evaluation_at_ingestion_is_the_invalid_requirement() { - use ExecutionTiming::{IngestionTime, QueryTime}; assert_eq!( - PhysicalRequirement::new(IngestionTime, Evaluation::Later), - Err(InvalidPhysicalRequirement::LaterEvaluationAtIngestion) + PhysicalRequirement::new(IngestionTime, Evaluation::Later, None), + Err(InvalidPhysicalProperties::LaterEvaluationAtIngestion) ); for (timing, evaluation) in [ (QueryTime, Evaluation::Same), (QueryTime, Evaluation::Later), (IngestionTime, Evaluation::Same), ] { - let requirement = PhysicalRequirement::new(timing, evaluation).unwrap(); - assert_eq!( - (requirement.reader_timing(), requirement.reads_from()), - (timing, evaluation) - ); + assert!(PhysicalRequirement::new(timing, evaluation, None).is_ok()); } } } diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs index 2c81ff371..eccdb87ca 100644 --- a/crates/types/tests/materialized_view.rs +++ b/crates/types/tests/materialized_view.rs @@ -2,6 +2,9 @@ //! (`asap_types::ir::materialized_view`): two one-minute KLL tumbling windows of `m` //! kept at ingestion, merged into a view refreshed on first read. +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -95,12 +98,20 @@ fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { for tumbling_window in &tumbling_windows { assignment.set( tumbling_window, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } assignment.set( &merge, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, + PhysicalProperties::new( + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&merge, &assignment).unwrap(); let dag = compile_physical_asap_dag(&merge, &physical_properties).unwrap(); @@ -232,7 +243,11 @@ fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { .map(|chain| { assignment.set( chain, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); QueryRoot::Operator( OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { @@ -371,7 +386,11 @@ fn a_shifted_tumbling_window_has_its_own_phase() { for tumbling_window in &tumbling_windows { assignment.set( tumbling_window, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } let roots: Vec = tumbling_windows @@ -418,7 +437,11 @@ fn views_of_unbounded_windows_are_groups_of_their_own() { .unwrap(); assignment.set( &build, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); QueryRoot::Operator(build) }) @@ -451,7 +474,11 @@ fn a_view_reading_a_view_of_its_group_follows_it() { for view in [&tumbling_window, &rollup] { assignment.set( view, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); } let roots = [QueryRoot::Operator(Rc::clone(&rollup))]; @@ -503,7 +530,11 @@ fn a_view_has_its_refresh_window_on_a_dag_without_edges() { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( &view, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, + PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"), ); let physical_properties = apply_physical_properties(&view, &assignment).unwrap(); let mut dag = compile_physical_asap_dag(&view, &physical_properties).unwrap(); diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 721835e9d..c04ffa78f 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -779,7 +779,21 @@ fn workload_export_matches_the_single_root_export() { #[test] fn export_validation_over_every_valid_state() { use ExecutionTiming::{IngestionTime, QueryTime}; - for properties in PhysicalProperties::every_valid() { + for properties in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] + .into_iter() + .flat_map(|timing| { + [ + Lifetime::None, + Lifetime::OneEvaluation(Medium::Memory), + Lifetime::OneEvaluation(Medium::Disk), + Lifetime::AcrossEvaluations(Medium::Memory), + Lifetime::AcrossEvaluations(Medium::Disk), + ] + .map(|lifetime| PhysicalProperties::new(timing, lifetime)) + }) + // Every valid state: the pairs `new` accepts. + .filter_map(Result::ok) + { let root = plan(); let build = root.children()[0]; let mut assignment = PhysicalPropertiesAssignment::default(); @@ -790,7 +804,9 @@ fn export_validation_over_every_valid_state() { } let timed = apply_physical_properties(&root, &assignment).unwrap(); let mut dag = compile_physical_asap_dag(&root, &timed).unwrap(); - let [scan, build, _finalize] = [0, 1, 2].map(|i| dag.nodes[i].physical_properties()); + let [scan, build, _finalize] = [0, 1, 2].map(|i| { + PhysicalProperties::new(dag.nodes[i].output_state.timing, dag.nodes[i].lifetime) + }); match (properties.timing(), properties.lifetime()) { (QueryTime, Lifetime::None) => { assert_eq!(build, Ok(properties)); @@ -809,7 +825,13 @@ fn export_validation_over_every_valid_state() { } (IngestionTime, Lifetime::AcrossEvaluations(_)) => { assert_eq!(build, Ok(properties)); - assert_eq!(scan, Ok(PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION)); + assert_eq!( + scan, + Ok( + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime") + ) + ); dag.validate().unwrap(); } (QueryTime, Lifetime::OneEvaluation(_)) => { diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 523a442f0..8d23fc81a 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,5 +1,6 @@ use asap_types::ir::operator::{AggIntent, Reduction, Source}; use asap_types::ir::properties::AccuracyGuarantee; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; @@ -367,7 +368,8 @@ fn population_timing_follows_its_physical_properties() { let mut evaluation_at_ingestion = PhysicalPropertiesAssignment::default(); evaluation_at_ingestion.set( &candidate, - PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION, + PhysicalProperties::new(ExecutionTiming::IngestionTime, Lifetime::None) + .expect("a valid timing and lifetime"), ); assert!(apply_physical_properties(&candidate, &evaluation_at_ingestion).is_err()); } diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index 279326129..877e19503 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -13,6 +13,8 @@ use asap_types::ir::{ apply_physical_properties, ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, PhysicalPropertiesAssignment, Predicate, ScalarExpr, }; +use asap_types::physical::Lifetime; +use asap_types::physical::Medium; use asap_types::physical::PhysicalProperties; use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are @@ -718,7 +720,11 @@ fn summary_merge_timing() { timings(&PhysicalPropertiesAssignment::default()), (QueryTime, vec![QueryTime, QueryTime]) ); - let at_ingestion = PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION; + let at_ingestion = PhysicalProperties::new( + ExecutionTiming::IngestionTime, + Lifetime::AcrossEvaluations(Medium::Memory), + ) + .expect("a valid timing and lifetime"); let mut stored_tumbling_windows = PhysicalPropertiesAssignment::default(); for tumbling_window in &tumbling_windows { stored_tumbling_windows.set(tumbling_window, at_ingestion);