From 8358a4dd7388e56b4dbca1e6ccaec57c8cd5defa Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:31:24 +0000 Subject: [PATCH 1/4] refactor(stage0, types): memo hash-consing and candidate storage (#1207 PR 2) --- crates/frontend-common/src/canonicalize.rs | 2 +- crates/stage0-memo/src/lib.rs | 496 ++++++++++++++++++- crates/types/src/ir/schema/schema_support.rs | 2 +- crates/types/src/stages/candidates.rs | 479 +++++++++++++++++- crates/types/src/stages/hash_consing.rs | 59 +++ crates/types/src/stages/memo.rs | 8 +- crates/types/src/stages/mod.rs | 1 + 7 files changed, 1016 insertions(+), 31 deletions(-) create mode 100644 crates/types/src/stages/hash_consing.rs diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 68a1eb47c..43495769f 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_stage0_memo`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 9311a60dd..80697373c 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -1,21 +1,493 @@ //! `asap-stage0-memo` — #509 Stage 0: the memo groups of the workload's //! query roots (a [`Memo`] without candidates). Each node of the queries' //! DAGs becomes the group of its expression, found or added by -//! [`Memo::find_or_add_group`], so identical sub-expressions are one group. -//! Stage 1 starts from these groups. +//! [`Memo::find_or_add_group`], so identical sub-expressions, within one +//! query or across queries, are one group. Stage 1 starts from these groups. //! -//! It depends only on `asap-types`, where its output (`Memo`) lives; no -//! other stage crate depends on it (the frontends' tests aside, listed in -//! `crate_graph.rs`), and the planner facade calls it first. +//! It depends only on `asap-types`, where its output (`Memo`) lives and +//! where the one expression equality is; no other stage crate depends on +//! it, and the planner facade calls it first. -use asap_types::ir::{QueryRoot, SchemaDerivationError}; -use asap_types::stages::Memo; +use std::collections::HashMap; +use std::rc::Rc; + +use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; +use asap_types::stages::{Memo, MemoGroupId}; /// Stage 0: the memo groups of the workload's query roots, query `i` -/// being `roots[i]`, every group without candidates. Fails with the first -/// node that breaks the node rules over its children's groups -/// ([`Memo::find_or_add_group`]). +/// being `roots[i]` (added by [`Memo::add_query`], so each group lists the +/// queries whose root reaches it), every group without candidates. Fails with the first node that breaks the +/// node rules over its children's groups ([`Memo::find_or_add_group`]); the +/// facade validates the roots first, so that is a frontend bug. pub fn memo_of_queries(roots: &[QueryRoot]) -> Result { - let _ = roots; - todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group") + let mut memo = Memo::default(); + let mut group_of_node = GroupOfNode::new(); + for root in roots { + for node in root.operator_refs() { + group_of(&mut memo, &mut group_of_node, node)?; + } + } + let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; + for root in roots { + memo.add_query(match root { + QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) + } + }); + } + Ok(memo) +} + +/// Each node of the queries' DAGs already visited, by address, to its +/// group: a node reached through several paths is visited once. The roots +/// outlive the map, so no address is reused. +type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; + +/// The group of `node`: its operator over its children's groups (each +/// found or added first), found or added by [`Memo::find_or_add_group`]. +fn group_of( + memo: &mut Memo, + group_of_node: &mut GroupOfNode, + node: &Rc, +) -> Result { + if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { + return Ok(group); + } + for child in node.children() { + group_of(memo, group_of_node, child)?; + } + let expression = node + .operator + .map_children(|child| group_of_node[&Rc::as_ptr(child)]); + let group = memo.find_or_add_group(expression)?; + group_of_node.insert(Rc::as_ptr(node), group); + Ok(group) +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::operator::agg_intent::AggIntent; + use asap_types::ir::operator::asap::ASAPOp; + use asap_types::ir::operator::operator_properties::{ + BinaryOpKind, GroupKeys, Reduction, Source, + }; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; + use asap_types::ir::schema::sketch::{ + GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate, + }; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; + + fn node(op: NonASAPOp) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() + } + + /// `[ts, service, value, latency]`, no unique key. + fn scan() -> Rc { + node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + fn quantile_agg_over( + child: Rc, + by: Vec, + col: Option, + q: f64, + accuracy: AccuracyTarget, + ) -> Rc { + node(NonASAPOp::Aggregate { + reduction: Reduction::by(by), + measures: vec![AggIntent::Quantile { col, q, accuracy }], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + } + + fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { + quantile_agg_over(scan(), by, col, q, AccuracyTarget::exact()) + } + + fn compare(lhs: Rc, rhs: Rc) -> Rc { + node(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Compare(CompareOpKind::Eq), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) + } + + /// The memo of `roots`, one query per root. + fn memo_of(roots: Vec>) -> Memo { + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + memo_of_queries(&roots).unwrap() + } + + /// Query `query`'s root group. + fn root(memo: &Memo, query: usize) -> MemoGroupId { + match &memo.query_roots[query] { + QueryRoot::Operator(group) => *group, + QueryRoot::Scalar(_) => panic!("expected an operator root"), + } + } + + /// Whether queries 0 and 1 of `[a, b]` are one group. + fn one_group(a: Rc, b: Rc) -> bool { + let memo = memo_of(vec![a, b]); + root(&memo, 0) == root(&memo, 1) + } + + /// Two queries sharing a sub-expression read one group for it, and + /// every group lists the queries that reach it and the groups that + /// read it. + #[test] + fn two_queries_sharing_a_sub_expression_get_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let p50_vs_p90 = compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.9), + ); + let memo = memo_of(vec![p50, p50_vs_p90]); + // scan, p50, p90, compare. + assert_eq!(memo.groups().len(), 4); + let (scan, p50, compare) = (MemoGroupId(0), root(&memo, 0), root(&memo, 1)); + let p90 = MemoGroupId(2); + assert_eq!( + memo.groups()[compare.0 as usize].expression.children(), + vec![&p50, &p90] + ); + assert_eq!(memo.groups()[scan.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p50.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p90.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[compare.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[scan.0 as usize].parent_groups(), &[p50, p90]); + assert_eq!(memo.groups()[p50.0 as usize].parent_groups(), &[compare]); + assert!(memo + .groups() + .iter() + .all(|group| group.candidates.is_empty())); + } + + /// One query's repeated sub-expression is one group, read twice. + #[test] + fn a_query_s_repeated_sub_expression_is_one_group() { + let memo = memo_of(vec![compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.5), + )]); + assert_eq!(memo.groups().len(), 3); + let p50 = MemoGroupId(1); + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .expression + .children(), + vec![&p50, &p50] + ); + assert_eq!( + memo.groups()[p50.0 as usize].parent_groups(), + &[root(&memo, 0)] + ); + } + + /// A node reached through two paths is one group, visited once. + #[test] + fn an_already_shared_node_is_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let memo = memo_of(vec![compare(Rc::clone(&p50), p50)]); + assert_eq!(memo.groups().len(), 3); + } + + /// Aggregates that differ in a column, a quantile or a filter are + /// different groups. + #[test] + fn aggregates_that_differ_in_an_argument_are_different_groups() { + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::Predicate; + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(3), 0.5) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5), + quantile_agg(vec![], Some(2), 0.9) + )); + let filtered = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Quantile { + col: Some(2), + q: 0.5, + accuracy: AccuracyTarget::exact(), + }], + output_names: vec![], + filters: vec![Some(Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: asap_types::ir::ExprSemantics::Sql, + }))], + having: None, + child: scan(), + }); + assert!(!one_group(quantile_agg(vec![1], Some(2), 0.5), filtered)); + } + + /// Equal nodes without a unique key are one group when they return the + /// same rows each time: a keyless scan, a global aggregate, a `without` + /// aggregate. + #[test] + fn equal_keyless_nodes_that_return_the_same_rows_are_one_group() { + let table_scan = || { + node(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + }) + }; + assert!(one_group(table_scan(), table_scan())); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9), + quantile_agg(vec![], Some(2), 0.9) + )); + let without = || { + node(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count { + accuracy: AccuracyTarget::exact(), + }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + }; + assert!(one_group(without(), without())); + } + + /// Quantiles that differ only in their accuracy target are two groups + /// over one scan group. + #[test] + fn differing_accuracy_targets_are_two_groups_over_one_scan_group() { + let quantile = |epsilon| { + quantile_agg_over( + scan(), + vec![1], + Some(2), + 0.99, + AccuracyTarget::max_error(epsilon), + ) + }; + let memo = memo_of(vec![quantile(0.01), quantile(0.05)]); + assert_ne!(root(&memo, 0), root(&memo, 1)); + assert_eq!(memo.groups().len(), 3); + } + + /// `vector(scalar(sum by (service) (m)))`. + fn scalar_of_vector() -> Rc { + let sum_up = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }); + node(NonASAPOp::PromqlVectorFromScalar( + ScalarExpr::PromqlScalarFromVector(sum_up), + )) + } + + /// A vector read through `scalar(v)` is a child group like any input, + /// one group across queries, and a scalar query's root reads it. + #[test] + fn a_scalar_referenced_vector_is_one_group_across_queries() { + let roots = vec![ + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let memo = memo_of_queries(&roots).unwrap(); + // scan, sum, bridge. + assert_eq!(memo.groups().len(), 3); + assert_eq!(root(&memo, 0), root(&memo, 1)); + let sum = MemoGroupId(1); + assert_eq!(memo.query_roots[2].operator_refs(), vec![&sum]); + assert_eq!( + memo.groups()[sum.0 as usize].reached_by_queries(), + &[0, 1, 2] + ); + } + + /// `SummaryAgg[DDSketch](scan)`. + fn summary_agg() -> Rc { + let family = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: scan(), + family, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + })) + .unwrap() + } + + /// Two quantile readouts of one summary are two groups over one state + /// group. + #[test] + fn readouts_share_their_state_group_but_not_each_other() { + let readout = |q| { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: summary_agg(), + query: SketchStatistic::Quantile { q }, + })) + .unwrap() + }; + let memo = memo_of(vec![readout(0.95), readout(0.99)]); + let (p95, p99) = (root(&memo, 0), root(&memo, 1)); + assert_ne!(p95, p99); + assert_eq!( + memo.groups()[p95.0 as usize].expression.children(), + memo.groups()[p99.0 as usize].expression.children() + ); + assert_eq!(memo.groups().len(), 4); + } + + /// `DISTINCT` on every column returns the same rows each time and is one + /// group; a `Dedup` on some columns keeps an unspecified row per key. + #[test] + fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + node(NonASAPOp::Dedup { + cols, + child: scan(), + }) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); + } + + fn unordered_limit(child: Rc) -> Rc { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child, + }) + } + + /// Two unordered `LIMIT 10`s may return different rows: two groups, and + /// so are equal parents over them. + #[test] + fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group(unordered_limit(scan()), unordered_limit(scan()))); + let distinct_over_limit = || { + node(NonASAPOp::Dedup { + cols: vec![], + child: unordered_limit(scan()), + }) + }; + assert!(!one_group(distinct_over_limit(), distinct_over_limit())); + } + + /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows + /// of a fixed order (one group); over a `Sort` partitioned differently, + /// which rows come first depends on input order (two groups). + #[test] + fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: node(NonASAPOp::Sort { + keys: vec![asap_types::ir::scalar::SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child: scan(), + }), + }) + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); + } + + /// A projection emitting the literal `value`. + fn literal(value: f64) -> Rc { + node(NonASAPOp::Project { + cols: vec![asap_types::ir::ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::literal_f64(value), + }], + qualifier: None, + child: scan(), + }) + } + + /// Signed zeros and different non-finite literals are different groups; + /// JSON's `null` for non-finite floats never decides equality. + #[test] + fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); + } + + /// A frontend node that breaks the node rules (a readout of raw rows + /// instead of summary state) makes Stage 0 return the error. + #[test] + fn memo_of_queries_returns_the_error_of_a_node_that_breaks_the_node_rules() { + let readout_of_rows = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: scan(), + query: SketchStatistic::Quantile { q: 0.5 }, + })) + .unwrap(); + assert!(memo_of_queries(&[QueryRoot::Operator(readout_of_rows)]).is_err()); + } } diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 94e997390..61bc53e69 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_stage0_memo`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 4779ffb7f..ec981edbd 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -3,8 +3,9 @@ //! $r^{\log}_{c,i}$ (concepts doc `memo-groups-properties-requirements.md`, //! section 3; #1203). -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; +use super::hash_consing::{expression_hash, same_expression, same_value}; use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; use crate::ir::operator::asap::family_of_operator; use crate::ir::properties::summary_coverage::SummaryCoverage; @@ -34,6 +35,10 @@ pub struct Memo { groups: Vec, /// Each query's root over groups, by index in the workload. pub query_roots: Vec>, + /// The groups [`Self::find_or_add_group`] may return, by + /// [`expression_hash`] of their expression: every group whose sharing + /// is legal ([`GroupProperties::shareable_when_equal`]). + groups_by_expression_hash: HashMap>, } impl Memo { @@ -46,34 +51,149 @@ impl Memo { /// groups and every group below them list it /// ([`MemoGroup::reached_by_queries`]). Stage 0 adds each query once. pub fn add_query(&mut self, root: QueryRoot) -> usize { - let _ = root; - todo!("#1207 PR 2") + let query = self.query_roots.len(); + let groups = root.operator_refs().into_iter().copied().collect(); + self.query_roots.push(root); + self.add_queries_below(groups, &[query]); + query } - /// Hash-consing: the group with this expression, added if new, with the - /// `GroupProperties` [`derive_group_properties`] derives. Fails when the - /// expression breaks the node rules (schema derivation, input - /// validation, merge check). + /// Hash-consing: the group whose expression is `expression` + /// (`same_expression`: operator, arguments and input groups), else a + /// new group for it, with the `GroupProperties` [`derive_group_properties`] + /// derives from its input groups, listed as a parent of each input + /// group. A group whose copies may return different rows (an unordered + /// `LIMIT`, [`GroupProperties::shareable_when_equal`]) is never returned + /// for another expression, so such an expression always gets a new + /// group. Fails, leaving the memo unchanged, when the expression breaks + /// the node rules (schema derivation, input validation, merge check). pub fn find_or_add_group( &mut self, expression: Operator, ) -> Result { - let _group_properties = derive_group_properties(self, &expression)?; - todo!("#1207 PR 2: hash-consing over groups") + let group_properties = derive_group_properties(self, &expression)?; + let hash = expression_hash(&expression); + let shareable = group_properties.shareable_when_equal; + if shareable { + if let Some(&group) = self + .groups_by_expression_hash + .get(&hash) + .and_then(|groups| { + groups.iter().find(|&&group| { + same_expression(&self.groups[group.0 as usize].expression, &expression) + }) + }) + { + return Ok(group); + } + } + let id = MemoGroupId( + u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), + ); + for &input in expression.children() { + add_parent(&mut self.groups[input.0 as usize], id); + } + self.groups.push(MemoGroup { + expression, + group_properties, + parents: Vec::new(), + queries: Vec::new(), + candidates: Vec::new(), + }); + if shareable { + self.groups_by_expression_hash + .entry(hash) + .or_default() + .push(id); + } + Ok(id) } - /// Adds `candidate` to `group`'s candidates. Fails when its expression - /// breaks the node rules ([`derive_group_properties`]) or its output - /// schema or kind differs from the group's. + /// Adds `candidate` to `group`'s candidates and returns its id, unless + /// the group already has a candidate with the same expression + /// (`same_expression`) and logical properties: then that candidate's + /// id, and `candidate` is dropped. A new candidate makes `group` a parent + /// of each of its input groups, and `group`'s queries reach each input + /// group and every group below it. + /// + /// Fails, leaving the memo unchanged, when the candidate's expression + /// breaks the node rules a group's does ([`derive_group_properties`]), + /// or when its output schema or kind differs from the group's: then it + /// does not compute the group's result. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let _output = derive_group_properties(self, &candidate.expression.operator) + let output = derive_group_properties(self, &candidate.expression.operator) .map_err(CandidateError::Schema)?; - let _ = group; - todo!("#1207 PR 2") + let group_properties = &self.groups[group.0 as usize].group_properties; + if output.schema != group_properties.schema + || output.result_kind != group_properties.result_kind + { + return Err(CandidateError::OutputDiffersFromGroup { group }); + } + let candidates = &mut self.groups[group.0 as usize].candidates; + if let Some(index_in_group) = candidates + .iter() + .position(|known| same_expression_and_logical_properties(known, &candidate)) + { + return Ok(CandidateForMemoGroupId { + group, + index_in_group, + }); + } + let input_groups: Vec = candidate + .expression + .operator + .children() + .into_iter() + .copied() + .collect(); + candidates.push(candidate); + let index_in_group = candidates.len() - 1; + for &input in &input_groups { + add_parent(&mut self.groups[input.0 as usize], group); + } + let queries = self.groups[group.0 as usize].queries.clone(); + self.add_queries_below(input_groups, &queries); + Ok(CandidateForMemoGroupId { + group, + index_in_group, + }) + } + + /// Adds `queries` to each of `groups` and every group below them, through + /// the groups' expressions and their candidates' input groups. A group + /// that has every query already is not walked: every group below it has + /// them too. + fn add_queries_below(&mut self, groups: Vec, queries: &[usize]) { + let mut to_visit = groups; + while let Some(group) = to_visit.pop() { + let group = &mut self.groups[group.0 as usize]; + let mut added = false; + for &query in queries { + if let Err(position) = group.queries.binary_search(&query) { + group.queries.insert(position, query); + added = true; + } + } + if added { + to_visit.extend( + group + .expression + .children() + .into_iter() + .chain( + group + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()), + ) + .copied(), + ); + } + } } /// The candidate `id`. @@ -184,6 +304,28 @@ pub enum CandidateError { OutputDiffersFromGroup { group: MemoGroupId }, } +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } +} + +/// Whether `left` and `right` are one candidate of a group: the same +/// expression ([`same_expression`]) and logical properties (parameters by +/// [`same_value`], window layout). +fn same_expression_and_logical_properties( + left: &CandidateForMemoGroup, + right: &CandidateForMemoGroup, +) -> bool { + same_expression(&left.expression.operator, &right.expression.operator) + && same_value( + &left.logical_properties.parameters, + &right.logical_properties.parameters, + ) + && left.logical_properties.window_layout == right.logical_properties.window_layout +} + /// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. #[derive(Debug, Clone)] pub struct CandidateForMemoGroup { @@ -1201,3 +1343,310 @@ mod tests { } } } + +/// The `Memo` methods: hash-consing groups ([`Memo::find_or_add_group`]) and +/// storing candidates ([`Memo::add_candidate`]), on the concepts doc's KLL +/// example (section 1: G1 the latency scan, G2 its KLL state). +#[cfg(test)] +mod memo_tests { + use super::*; + use crate::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::{DataType, Field, Schema, SketchStatistic, SummaryUpdate}; + use crate::ir::NonASAPOp; + + /// `Scan metric[1m]`: `[ts, value]`. + fn scan(metric: &str) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + /// `SummaryAgg[KLL](input)`: no θ in the expression. + fn kll_state(input: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: input, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }) + } + + /// `SummaryEstimate[q](state)`. + fn quantile_readout(state: MemoGroupId, q: f64) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }) + } + + /// An unordered `LIMIT 10` over `input`, whose copies may return + /// different rows. + fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) + } + + /// `Project(input)` keeping its first column. + fn project_first_column(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![crate::ir::ProjectItem { + alias: None, + expr: crate::ir::ScalarExpr::Column(0), + }], + qualifier: None, + child: input, + }) + } + + /// A candidate computing `operator` with `parameters` and no window. + fn candidate( + operator: Operator, + parameters: Option, + ) -> CandidateForMemoGroup { + let input_requirements = vec![LogicalRequirements::new(None); operator.children().len()]; + CandidateForMemoGroup { + // Built directly: the doc example's k = 200 and k = 345 are not + // in today's KLL list, which `LogicalProperties::new` checks. + logical_properties: LogicalProperties { + parameters, + window_layout: None, + }, + expression: CandidateExpression { operator }, + input_requirements, + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + } + } + + /// KLL `k` (no recorded failure probability). + fn kll_k(k: u32) -> Option { + Some(SketchParams::Kll { + k, + failure_probability: None, + }) + } + + /// An equal expression over the same input groups is found, not added. + #[test] + fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.find_or_add_group(scan("latency")).unwrap(), g1); + assert_eq!(memo.find_or_add_group(kll_state(g1)).unwrap(), g2); + assert_ne!(memo.find_or_add_group(scan("errors")).unwrap(), g1); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); + } + + /// The same operator over different input groups is a different group. + #[test] + fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::default(); + let latency = memo.find_or_add_group(scan("latency")).unwrap(); + let errors = memo.find_or_add_group(scan("errors")).unwrap(); + assert_ne!( + memo.find_or_add_group(kll_state(latency)).unwrap(), + memo.find_or_add_group(kll_state(errors)).unwrap() + ); + } + + /// An expression whose copies may return different rows (an unordered + /// `LIMIT`) gets a new group every time. + #[test] + fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); + } + + /// A new group is a parent of each of its input groups, listed once. + #[test] + fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.groups()[g1.0 as usize].parent_groups(), &[g2]); + assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); + } + + /// KLL builds with k = 200 and k = 345 are two candidates of one group + /// G2: θ is a logical property, not part of the expression. + #[test] + fn kll_builds_with_k_200_and_k_345_share_one_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k200 = memo + .add_candidate(k200_group, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let k345_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k345 = memo + .add_candidate(k345_group, candidate(kll_state(g1), kll_k(345))) + .unwrap(); + assert_eq!(k200_group, k345_group); + assert_ne!(k200, k345); + assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k345).logical_properties.parameters(), + kll_k(345).as_ref() + ); + } + + /// A candidate with the same expression and logical properties as one + /// in the group is not added again; its id is returned. + #[test] + fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let first = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let again = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let mut windowed = candidate(kll_state(g1), kll_k(200)); + windowed.logical_properties.window_layout = + Some(WindowLayout::Tumbling { width_ms: 10_000 }); + assert_ne!(memo.add_candidate(g2, windowed).unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); + } + + /// A new candidate makes its group a parent of each group it reads, and + /// the group's queries reach those groups and every group below them. + #[test] + fn add_candidate_maintains_parents_and_queries() { + // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + assert_eq!(memo.add_query(QueryRoot::Operator(g3)), 0); + // A rule adds a state group G5 over another scan G4: no query + // reaches them yet. + let g4 = memo.find_or_add_group(scan("latency_copy")).unwrap(); + let g5 = memo.find_or_add_group(kll_state(g4)).unwrap(); + assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); + // G3 gains a candidate reading G5: G5 and G4 now serve query 0. + memo.add_candidate(g3, candidate(quantile_readout(g5, 0.95), None)) + .unwrap(); + assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); + assert_eq!(memo.groups()[g4.0 as usize].parent_groups(), &[g5]); + assert_eq!(memo.groups()[g5.0 as usize].reached_by_queries(), &[0]); + assert_eq!(memo.groups()[g4.0 as usize].reached_by_queries(), &[0]); + // Query 1 is G6 = Project(G3): it reaches G3 and everything below + // it, through expressions and candidates, in ascending order. + let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); + assert_eq!(memo.add_query(QueryRoot::Operator(g6)), 1); + for group in [g1, g2, g3, g4, g5] { + assert_eq!( + memo.groups()[group.0 as usize].reached_by_queries(), + &[0, 1], + "group {group:?}" + ); + } + assert_eq!(memo.groups()[g3.0 as usize].parent_groups(), &[g6]); + } + + /// A candidate that computes another result than its group's (a + /// readout offered for a state group) is rejected and changes nothing. + #[test] + fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g2, candidate(quantile_readout(g2, 0.5), None)), + Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); + } + + /// A candidate whose expression fails input validation (a readout of + /// raw rows instead of summary state) is rejected and changes nothing. + #[test] + fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| { + MemoGroupAsInput { + memo: &memo, + group_id, + } + }); + assert!( + over_raw_rows.output_schema().is_ok(), + "fixture: the schema derives; only input validation rejects it" + ); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g3, candidate(quantile_readout(g1, 0.95), None)), + Err(CandidateError::Schema(_)) + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); + } + + /// An expression with no output schema (a group-by column out of range) + /// is an error, and no group is added. + #[test] + fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![9]), + measures: vec![crate::ir::operator::agg_intent::AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child: g1, + }); + assert!(memo.find_or_add_group(out_of_range).is_err()); + assert_eq!(memo.groups().len(), 1); + assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); + } +} diff --git a/crates/types/src/stages/hash_consing.rs b/crates/types/src/stages/hash_consing.rs new file mode 100644 index 000000000..3a482dda0 --- /dev/null +++ b/crates/types/src/stages/hash_consing.rs @@ -0,0 +1,59 @@ +//! The one equality that makes two expressions one memo group, and two +//! candidates of a group one candidate: [`same_expression`] (operator, +//! arguments and input groups). [`Memo::find_or_add_group`] looks a group up +//! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by +//! it. Parameters θ are not an argument of any operator (they are a +//! candidate's `LogicalProperties`), so builds of one family over one input +//! that differ only in θ are one group. +//! +//! [`Memo::find_or_add_group`]: super::Memo::find_or_add_group +//! [`Memo::add_candidate`]: super::Memo::add_candidate + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::MemoGroupId; +use crate::ir::Operator; + +/// `expression` with every input group replaced by `()`: the operator and +/// its arguments. +fn operator_and_arguments(expression: &Operator) -> Operator<()> { + expression.map_children(|_| ()) +} + +/// Whether `left` and `right` are one expression: equal operators and +/// arguments ([`same_value`]) over the same input groups at the same +/// positions. +pub(super) fn same_expression(left: &Operator, right: &Operator) -> bool { + left.children() == right.children() + && same_value( + &operator_and_arguments(left), + &operator_and_arguments(right), + ) +} + +/// A hash consistent with [`same_expression`], used only to narrow the +/// groups compared: SipHash over the canonical JSON of the operator and its +/// arguments, then the input groups. A non-finite `f64` serializes as +/// `null`, which only widens one bucket; [`same_expression`] decides. +pub(super) fn expression_hash(expression: &Operator) -> u64 { + let mut hasher = DefaultHasher::new(); + serde_json::to_string(&operator_and_arguments(expression)) + .unwrap_or_default() + .hash(&mut hasher); + expression.children().hash(&mut hasher); + hasher.finish() +} + +/// Typed equality, confirmed by equal serializations: numeric `PartialEq` +/// alone conflates signed zeros, and `SketchParams`' equality omits the +/// recorded error and failure probability. The serialized check is +/// additional evidence, never a replacement for typed equality (JSON maps +/// non-finite floats to `null`). +pub(super) fn same_value(left: &T, right: &T) -> bool { + left == right + && match (serde_json::to_string(left), serde_json::to_string(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index e7c3948cd..6678a0020 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -25,10 +25,14 @@ pub struct MemoGroup { /// changes. pub group_properties: GroupProperties, /// The groups that read it ([`Self::parent_groups`]) through their - /// expression or a candidate's. + /// expression or a candidate's: kept up to date by + /// [`Memo::find_or_add_group`](super::Memo::find_or_add_group) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub parents: Vec, /// The queries whose roots reach it ([`Self::reached_by_queries`]), - /// ascending. + /// ascending: kept up to date by + /// [`Memo::add_query`](super::Memo::add_query) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 89f454ada..0d55cd1f3 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -8,6 +8,7 @@ //! live here. mod candidates; +mod hash_consing; mod logical_memo_candidates; mod memo; mod per_computation_logical_memo_candidates; From b3abbbad381585a44008bf0ed9d80c9049dc3068 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:37:46 +0000 Subject: [PATCH 2/4] =?UTF-8?q?test(types):=20memo=20summary=20coverage;?= =?UTF-8?q?=20asap-primitive-schema=20=C2=A74.2.2=20over=20memo=20groups?= =?UTF-8?q?=20(#1207=20PR=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/types/src/stages/candidates.rs | 8 +- crates/types/tests/main.rs | 1 + crates/types/tests/memo_summary_coverage.rs | 406 ++++++++++++++++++ .../proposals/asap-primitive-schema.md | 187 +++++--- 4 files changed, 530 insertions(+), 72 deletions(-) create mode 100644 crates/types/tests/memo_summary_coverage.rs diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index ec981edbd..bb9413413 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -1609,11 +1609,9 @@ mod memo_tests { let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); - let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| { - MemoGroupAsInput { - memo: &memo, - group_id, - } + let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| MemoGroupAsInput { + memo: &memo, + group_id, }); assert!( over_raw_rows.output_schema().is_ok(), diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index bf56559c0..8c1477bf9 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_summary_coverage; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs new file mode 100644 index 000000000..cf96a6c45 --- /dev/null +++ b/crates/types/tests/memo_summary_coverage.rs @@ -0,0 +1,406 @@ +//! Group properties and summary coverage, derived once per memo group from +//! its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` +//! §4.2.2). The memo of the worked example: +//! +//! ```text +//! G1 Scan latency +//! G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) +//! G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) +//! G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) +//! ``` +use std::ops::Bound; +use std::time::Duration; + +use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; +use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, CoverageError, SummaryCoverage, +}; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ + DataType, Field, FieldDataType, GroupingStrategy, MergeInputRule, Schema, SketchAlgorithm, + SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, + SchemaDerivationError, TimeRangeKind, +}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::test_support::definition_id; + +const JOB: usize = 2; +const A: usize = 3; +const B: usize = 4; +const MINUTE_MS: i64 = 60_000; + +/// `Scan latency{predicates}`: `[ts, value, job, a, b]`. +fn scan(predicates: Vec>) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: predicates.into_iter().map(Predicate).collect(), + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + Field::plain("a", DataType::Float64, false), + Field::plain("b", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +fn job_is(job: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(JOB)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(job.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `a + b > 10`: it spans two columns, so it is no selection. +fn a_plus_b_above_10() -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::Column(A)), + right: Box::new(ScalarExpr::Column(B)), + semantics: ExprSemantics::Sql, + }), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + } +} + +fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +fn range(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms as u64), + kind: TimeRangeKind::Range, + child, + }) +} + +/// `SummaryAgg[KLL]` of the sample value per series. +fn kll(child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }) +} + +fn merge(children: Vec) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + newest_tumbling_windows: None, + group_by: None, + }) +} + +/// KLL over the last minute of `G1` filtered by `pred`. +fn kll_of_last_minute_where( + memo: &mut Memo, + g1: MemoGroupId, + pred: ScalarExpr, +) -> MemoGroupId { + let filtered = memo.find_or_add_group(filter(pred, g1)).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, filtered)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() +} + +fn coverage(memo: &Memo, group: MemoGroupId) -> &SummaryCoverage { + memo.groups()[group.0 as usize] + .group_properties + .summary_coverage + .as_ref() + .expect("a summary state") +} + +/// `KLL` as an unlifted operator: its main input is `None`. +fn kll_unlifted() -> Operator> { + kll(MemoGroupId(0)).map_children(|_| None) +} + +fn jobs(values: &[&str]) -> Constraint { + Constraint::In( + values + .iter() + .map(|value| ScalarValue::Utf8((*value).into())) + .collect(), + ) +} + +fn last_minute() -> Option<(Bound, Bound)> { + Some((Bound::Excluded(-MINUTE_MS), Bound::Included(0))) +} + +/// A group's properties come from its operator and its input groups' +/// properties: the scan's schema, the state's kind and state column, the +/// readout's value column, and whether equal copies may be one group. +#[test] +fn group_properties_come_from_input_groups() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let readout = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Quantile { q: 0.99 }, + })) + .unwrap(); + let limit = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(1), + offset: 0, + partition_by: GroupKeys::default(), + child: g1, + })) + .unwrap(); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + assert_eq!(properties(g1).schema.fields.len(), 5); + assert_eq!(properties(g1).summary_coverage, None); + assert_eq!(properties(g4).result_kind, OperatorResultKind::State); + assert!(properties(g4) + .schema + .fields + .iter() + .any(|field| field.name == "state" && !field.is_plain())); + assert!(properties(g4).summary_coverage.is_some()); + assert_eq!( + properties(readout).result_kind, + OperatorResultKind::InstantVector + ); + assert!(properties(readout) + .schema + .fields + .iter() + .any(|field| field.name == "quantile" + && field.dtype == FieldDataType::Plain(DataType::Float64))); + assert!(properties(g4).shareable_when_equal); + assert!(!properties(limit).shareable_when_equal); +} + +/// G4: `job = "api"` and the minute are lifted; the definition is the KLL +/// over G1. +#[test] +fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![kll_unlifted()] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.relative_time(), last_minute()); +} + +/// G7 has G4's definition and id and a disjoint selection, so +/// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. +#[test] +fn states_with_equal_definitions_and_disjoint_selections_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); + assert_eq!( + coverage(&memo, g4).definition(), + coverage(&memo, g7).definition() + ); + assert_eq!( + definition_id(coverage(&memo, g4)), + definition_id(coverage(&memo, g7)) + ); + let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); + let coverage = coverage(&memo, merged); + assert_eq!(coverage.definition().input_group, Some(g1)); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// Two states that select the same rows do not merge: KLL counts a row +/// once per input. +#[test] +fn states_with_overlapping_selections_do_not_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let groups = memo.groups().len(); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g4])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::Disjoint + } + )) + )); + assert_eq!(memo.groups().len(), groups); +} + +/// G10: `a + b > 10` spans two columns, so it stays as a `Filter` in the +/// definition; G10 does not merge with G4 although both select job = api. +#[test] +fn a_filter_that_is_no_selection_stays_unlifted() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g10 = kll_of_last_minute_where( + &mut memo, + g1, + ScalarExpr::BoolAnd(vec![job_is("api"), a_plus_b_above_10()]), + ); + let coverage_of_g10 = coverage(&memo, g10); + assert_eq!(coverage_of_g10.definition().input_group, Some(g1)); + assert_eq!( + coverage_of_g10.definition().unlifted_operators, + vec![ + kll_unlifted(), + filter(a_plus_b_above_10(), MemoGroupId(0)).map_children(|_| None), + ] + ); + assert_eq!( + coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g10])), + Err(SchemaDerivationError::Coverage( + CoverageError::DefinitionMismatch + )) + )); +} + +/// `a > (SELECT threshold)`: a scalar subquery reading group `threshold`. +fn a_above(threshold: MemoGroupId) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(A)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::ScalarSubquery(threshold)), + semantics: ExprSemantics::Sql, + } +} + +/// A residual `Filter` that reads a scalar subquery names the subquery's +/// group (`Some`) next to its main input (`None`). Two states with that +/// equal definition and disjoint selections merge. +#[test] +fn states_whose_definitions_read_the_same_subquery_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let threshold = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { + rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(10.0))]], + schema: Schema::new(vec![Field::plain("threshold", DataType::Float64, false)]), + })) + .unwrap(); + let state_of_job = |memo: &mut Memo, job: &str| { + let pred = ScalarExpr::BoolAnd(vec![job_is(job), a_above(threshold)]); + kll_of_last_minute_where(memo, g1, pred) + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, Some(g1)); + assert_eq!( + definition.unlifted_operators, + vec![ + kll_unlifted(), + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(a_above(threshold).map_operator_refs(&mut |group| Some(*group))), + child: None, + }), + ] + ); + assert_eq!(definition, coverage(&memo, web).definition()); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// PromQL label matchers are the scan's predicates: they are lifted too, so +/// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, +/// without them, is the last unlifted operator. +#[test] +fn scan_predicates_lift_and_their_states_merge() { + let mut memo = Memo::default(); + let state_of_job = |memo: &mut Memo, job: &str| { + let series = memo.find_or_add_group(scan(vec![job_is(job)])).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, None); + assert_eq!( + definition.unlifted_operators, + vec![kll_unlifted(), scan(vec![]).map_children(|_| None)] + ); + assert_eq!( + coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// A tumbling-window chain of five minutes covers what one build over the +/// five-minute range does: the same definition and selection. +#[test] +fn a_tumbling_window_chain_covers_its_whole_window() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let chain = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: g1, + family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + tumbling_window_width_ms: MINUTE_MS as u64, + tumbling_window_count: 5, + })) + .unwrap(); + let five_minutes = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let direct = memo.find_or_add_group(kll(five_minutes)).unwrap(); + assert_eq!(coverage(&memo, chain), coverage(&memo, direct)); + assert_eq!( + coverage(&memo, chain).selection()[0].relative_time(), + Some((Bound::Excluded(-5 * MINUTE_MS), Bound::Included(0))) + ); +} diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 15ff0f4ed..19920823c 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -112,10 +112,10 @@ A node in the physical data will represent the data or summary instance, so a no | Why not put it in the schema? | States worth merging cover different data but must have the same schema | below | | What is it based on? | Goldstein & Larson view matching (SIGMOD 2001), explained with an example | §4.1 | | What does it store? | `definition` (what is computed) + `selection` (which rows were taken) | §4.2.1 | -| How is it computed? | by the planner, from the sub-DAG the node covers, in one walk that yields both the definition and the selection | §4.2.2 | -| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; cached per node | §4.2.3 | +| How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | +| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `OperatorNode::coverage()`, `SummaryCoverage::derive` | §6.1, §6.6 | +| Where is it in the code? | `GroupProperties::summary_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -235,70 +235,109 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -The planner computes the coverage of a node from the sub-DAG the node covers, not from a declaration. One walk down the sub-DAG produces both parts: every filter condition either moves into `selection` or stays in `definition`. The definition is described first, then how the planner decides which conditions move into the selection. +Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates a summary-state group, `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::summary_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint. -**Worked example.** p99 latency per job over table `t` (columns `job`, `region`, `latency`): +```rust +/// What a summary state computes, with every lifted selection removed. +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ + /// / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. + pub unlifted_operators: Vec>>, +} -```sql -SELECT job, - approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) -FROM (SELECT job, region AS r, latency - FROM t - WHERE region = 'us' AND latency * 2 > 10) -GROUP BY job; +pub struct SummaryCoverage { + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl SummaryCoverage { + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + /// A hash of the definition: equal definitions of one memo get equal ids. + pub(crate) fn definition_id(&self) -> String; +} ``` -The SQL frontend lowers it as follows: `FILTER (WHERE …)` becomes the aggregate's filter, `region AS r` becomes a `Project`, and the `WHERE` conditions are folded into the `Scan`'s predicates. With an error target, the planner answers the quantile from a KLL `SummaryAgg`, read out by a `SummaryEstimate`. The sub-DAG under that `SummaryEstimate`: +**Worked example.** Three queries over the series `latency` (labels `job`, `a`, `b`), each a KLL state over the last minute: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job, filter: latency < 100 - ▲ - │ - [ Project ] job, region AS r, latency - ▲ - │ - [ Scan t ] predicates: region = 'us' AND latency * 2 > 10 +G1 Scan latency +G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) ← Q1 +G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) ← Q2 +G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) ← Q3 ``` -##### The definition - +**Case 1, G4: everything is lifted.** The walk starts at G4's `SummaryAgg` and reads each input group's `expression`: -The `definition` is the `SummaryAgg` together with its sub-DAG, with every condition that moves into `selection` (below) taken out. Everything else stays exactly as it is: +| Step | Group | Expression | What the walk does | +|---|---|---|---| +| 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | +| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as time `(−60 s, 0]` | +| 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | +| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | -| In the sub-DAG | In the `definition` | -|---|---| -| a `Filter` whose conditions all move into `selection` | removed | -| a `Filter` with some conditions that stay | kept, with only the conditions that stay | -| `Scan.predicates` and `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | -| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | -| any other operator | unchanged | +```text +G4.summary_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL]], + }, + selection: [ { job ∈ {api}, time (−60 s, 0] } ], +} +``` -So the `definition` holds what the state computes: the computation `C` with its remaining conditions, the summary family and its parameters, the input column, and the grouping `G`. +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, time (−60 s, 0] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, time (−60 s, 0] }`. -In the worked example, `latency < 100` and `region = 'us'` move into `selection` (below), and `latency * 2 > 10` stays: +**Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job ← filter removed - ▲ - │ - [ Project ] job, region AS r, latency ← unchanged - ▲ - │ - [ Scan t ] predicates: latency * 2 > 10 ← region = 'us' removed +G10.summary_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], + }, + selection: [ { job ∈ {api}, time (−60 s, 0] } ], +} ``` -**When two definitions are equal.** Merging (§5.6) requires equal definitions. +Its definition differs from G4's, so G10 cannot merge with G4, even though both select `job = api`. + +**The fields.** + +- `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). +- `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. +- `selection`: which output rows of that computation the state took, as a union of boxes. +- `definition_id()`: a hash of the definition, the key a summary store files states under together with a concrete selection. + +##### The definition + +The `definition` is what remains of the operators the walk passed, with every condition that moves into `selection` (below) taken out: + +| On the walk | In `unlifted_operators` | +|---|---| +| a `Filter` whose conditions all move into `selection` | removed | +| a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | +| `SummaryAgg.filter` | trimmed the same way | +| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | +| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | +| a direct-column `Project` | unchanged | +| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | +| any other operator (the first group the walk cannot pass) | not in the list: it is the `input_group` | -- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. -- `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. -- States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. +A `SummaryTumblingWindowChain` covers what one build over its whole window does: the walk starts at that build's `SummaryAgg` over a range `TimeRange` of the whole window over the chain's input. A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. + +**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input_group` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. ##### The selection @@ -307,7 +346,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **Steps.** -1. **Collect the conditions.** Go down the sub-DAG from the `SummaryAgg` and collect every filter condition: from `Filter` nodes, from `Scan.predicates`, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. +1. **Collect the conditions.** Go down the input groups from the `SummaryAgg` and collect every filter condition: from `Filter` groups, from the `Scan.predicates` where the walk stops, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. 2. **Ask two questions about each condition:** - **Rule 1: would it pick the same rows if it were moved to just below the `SummaryAgg`?** `region = 'us'` below a `Project` that only renames columns: yes. `value > 5` below `rate`: no, because it filters the raw samples that `rate` reads, which changes the rate values. - **Rule 2: is it a simple condition on one column?** That is, a value set such as `region IN ('us', 'eu')`, or a range such as `latency < 100`. `latency * 2 > 10` is not: it is on an expression. @@ -316,7 +355,16 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` - If either answer is no: leave it in the sub-DAG, so it is part of `definition`. The paper calls such conditions *residuals*. - One more case stays in `definition`: a condition on a column that already has a lifted condition it cannot be combined with (a forbidden-value set and a range, or values of different types). Conditions are taken in walk order: the `SummaryAgg.filter`, then the `Filter`s from top to bottom, then `Scan.predicates`. -In the worked example: +A SQL example, p99 latency per job over table `t` (columns `job`, `region`, `latency`), whose `WHERE` conditions the SQL front end folds into the `Scan`'s predicates and whose `region AS r` is a `Project`: + +```sql +SELECT job, + approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) +FROM (SELECT job, region AS r, latency + FROM t + WHERE region = 'us' AND latency * 2 > 10) +GROUP BY job; +``` | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| @@ -403,7 +451,7 @@ Any other shape stays in `definition`: #### 4.2.3 Cost of deriving coverage -**When it runs.** `coverage()` derives a node's coverage the first time it is called and caches it on the node, so each node pays once. Building a `SummaryMerge` (`OperatorNode::new`) derives its coverage to reject an invalid merge and caches the result, so `coverage()` and `validate_structure` do not derive it again. `validate_structure` derives it only for a merge that was not built through `new`. +**When it runs.** `find_or_add_group` derives a summary-state group's coverage once, when it creates the group, and stores it in `GroupProperties::summary_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. **Sizes used below.** @@ -886,9 +934,10 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchKind, GroupingStrategy) `state` field has type `family`) │ └── algorithm, params §6.4 -├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -└── coverage() → SummaryCoverage §6.6 - └── definition: OperatorNode (again a node, so the same structure) +└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) + +MemoGroup.group_properties.summary_coverage: SummaryCoverage §6.6 + └── definition: SummaryStateDefinition { input_group, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -898,7 +947,7 @@ OperatorNode §6.1 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.family` | | §6.4 Summary-family parameters | `SketchKind`, `SketchParams`, `GroupingStrategy`, exact/sample/wavelet/model params | the payload of each non-plain `FieldDataType` variant | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | +| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::summary_coverage` of summary-state memo groups | **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `family = Sketch(SketchKind::new(Kll, Kll { k: 200 }), PerSubpopulationInstance)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5). Its `schema` (§6.3) is `(job Utf8, state Sketch(KLL k=200))`: the `state` field has the same type as `family`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). @@ -1339,16 +1388,16 @@ A finalized value's accuracy statement is `AccuracyGuarantee` (`ir/properties/gu ### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) -The code for §4, as of #646. On `main` today `SummaryCoverage` is still the older declared form (`source` plus `regions`). +The code for §4. `SummaryStateDefinition` is shown in §4.2.2. ```rust pub struct SummaryCoverage { - /// What the state computes: the `SummaryAgg` and its sub-DAG, with the - /// conditions that went into `selection` taken out (§4.2.2). - pub definition: Rc, + /// What the state computes: its input group and the operators the walk + /// did not lift (§4.2.2). + pub(crate) definition: SummaryStateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. - pub selection: Vec, + pub(crate) selection: Vec, } /// One box: a row is in it when it meets every constraint. @@ -1400,9 +1449,10 @@ pub enum IntervalConstraintLimit { ```rust impl SummaryCoverage { - /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). - /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, - /// `SummaryTumblingWindowChain` or `SummaryMerge`; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` + /// The coverage of a summary-state group computed by `operator` over its + /// input groups (§4.2.2). Errors: `NotSummary` for an operator that is + /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a + /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a /// merge, an input's own error, `EmptyMerge` (no inputs), /// `DefinitionMismatch` (inputs compute different things) or @@ -1412,9 +1462,12 @@ impl SummaryCoverage { /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). - pub fn derive(node: &OperatorNode) -> Result; + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + pub(crate) fn definition_id(&self) -> String; } ``` -- `OperatorNode::new` and `validate_structure` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap; #592 relaxes the overlap check per family). The derived coverage is cached on the node. +- `find_or_add_group` and `add_candidate` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap, per the family's `MergeInputRule`). Operator-node trees have no coverage and no merge check. - A `SummaryAgg` always has coverage: what cannot go into `selection` stays in `definition`. From ef0e7747dac8b54b0d33dac6b32a6c2a093694d5 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:34:55 +0000 Subject: [PATCH 3/4] feat(types): coverage of time selections, chains and exact aggregates; @ start() as @ end() offset D (#1207 PR 2) Timestamp constraints use IntervalConstraintLimit: the one range TimeRange and the TimeShifts on a path lift into [anchor - o - W, anchor - o) on the timestamp column, anchored at the evaluation time (also @ end()) or at T (@ T). The PromQL front end lowers @ start() as @ end() with the query range's duration more offset; AtModifier::Start is deleted. SQL folds a timestamp cast of a literal into a Timestamp literal and resolution drops casts to the operand's own type, so absolute timestamp filters lift. A chain's coverage is a build's over its input; a TimeRange/TimeShift over a chain constrains the timestamp column of the chain's input, so SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X)))) covers what SummaryAgg(TimeRange(W, TimeShift(o, X))) does. Counter states merge only over intervals disjoint in time. Exact Aggregate groups get coverage too. --- .../executor/src/operators/series_window.rs | 5 +- crates/executor/src/runtime/context.rs | 2 +- crates/frontend-common/src/resolve.rs | 20 +- crates/frontend-promql/src/lib.rs | 39 ++- crates/frontend-promql/src/promql.rs | 72 ++-- .../tests/promql_conformance.rs | 4 +- crates/frontend-sql/src/sql/expr.rs | 21 +- crates/frontend-sql/src/sql/types.rs | 7 + crates/frontend-sql/tests/sql_lowering.rs | 32 ++ .../src/ir/operator/operator_properties.rs | 2 - .../src/ir/properties/summary_coverage.rs | 313 +++++++++++++---- crates/types/src/stages/candidates.rs | 20 +- crates/types/src/stages/memo.rs | 7 +- crates/types/tests/memo_summary_coverage.rs | 320 ++++++++++++++++-- .../proposals/asap-primitive-schema.md | 94 ++--- 15 files changed, 756 insertions(+), 202 deletions(-) diff --git a/crates/executor/src/operators/series_window.rs b/crates/executor/src/operators/series_window.rs index 6d81f48df..f691886e5 100644 --- a/crates/executor/src/operators/series_window.rs +++ b/crates/executor/src/operators/series_window.rs @@ -139,11 +139,10 @@ pub(super) fn anchor( match bound { None => Ok(literal), Some(bound) => { - let (start, end) = context + let (_, end) = context .query_range() - .ok_or_else(|| invalid("@ start() and @ end() require query range bounds"))?; + .ok_or_else(|| invalid("@ end() requires query range bounds"))?; match bound { - AtModifier::Start => Ok(Some(start)), AtModifier::End => Ok(Some(end)), AtModifier::Timestamp(_) => Err(invalid("invalid range-bound anchor")), } diff --git a/crates/executor/src/runtime/context.rs b/crates/executor/src/runtime/context.rs index 9714bda91..51a865082 100644 --- a/crates/executor/src/runtime/context.rs +++ b/crates/executor/src/runtime/context.rs @@ -62,7 +62,7 @@ impl RunContext { }), }) } - /// Supply the outer range query bounds for PromQL `@ start()` / `@ end()` + /// Supply the outer range query bounds for PromQL `@ end()` /// and for `@`-pinned calls, which are evaluated once at the query start. /// A range query run step by step must always supply them; without bounds /// each step is treated as its own instant query. For an instant query, diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 1fb82f5a9..8a89db9e5 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -487,11 +487,21 @@ fn resolve_expr_in( S::Not(e) => ScalarExpr::Not(bx(e)?), S::IsNull(e) => ScalarExpr::IsNull(bx(e)?), S::IsNotNull(e) => ScalarExpr::IsNotNull(bx(e)?), - S::Cast { expr, to, try_cast } => ScalarExpr::Cast { - expr: bx(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, + // A cast to the type its operand already has is the operand: SQL's + // `ts >= '2026-11-01'` casts the Timestamp column `ts` to DataFusion's + // nanosecond timestamp, which is the same IR type. + S::Cast { expr, to, try_cast } => { + let operand = bx(expr)?; + if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + *operand + } else { + ScalarExpr::Cast { + expr: operand, + to: to.clone(), + try_cast: *try_cast, + } + } + } S::InList { expr, list, diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index 38f9db6b0..9c5f65023 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -107,6 +107,9 @@ pub fn lower_promql_query_workload( &entry.requirements.accuracy.clone(), &sample_interval_of, &metric_type_of, + entry + .range_evaluation + .map_or(0, |range| range.span_ms as i64), )?; Ok(root) }) @@ -118,11 +121,12 @@ mod tests { use std::rc::Rc; use std::time::Duration; + use asap_types::ir::operator::operator_properties::{AtModifier, TimeShift}; use asap_types::ir::operator::Source; - use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; + use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, SourceFacts, WorkloadError, + QueryWorkload, RangeEvaluation, SourceFacts, WorkloadError, }; use super::*; @@ -367,4 +371,35 @@ mod tests { ) .is_empty()); } + + /// `@ start()` of a range query that spans 1 h is `@ end() offset 1h`: + /// the range starts 1 h before it ends. + #[test] + fn at_start_is_at_end_offset_by_the_query_range() { + let mut workload = workload("sum_over_time(data[5m] @ start() offset 1m)"); + if let Some(batch) = workload.query_workload.query_batch.as_mut() { + batch[0].range_evaluation = Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }); + } + let roots = lower_promql_query_workload(&workload).unwrap(); + let [QueryRoot::Operator(root)] = &roots[..] else { + panic!("one operator root"); + }; + let shifts: Vec = OperatorNode::reachable(root) + .iter() + .filter_map(|node| match node.non_asap()? { + NonASAPOp::TimeShift { shift, .. } => Some(*shift), + _ => None, + }) + .collect(); + assert_eq!( + shifts, + [TimeShift { + offset_ms: 3_660_000, + at: Some(AtModifier::End), + }] + ); + } } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 6c8a33f2a..86f4fea91 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -97,6 +97,9 @@ pub(crate) struct PromqlLowerer<'a> { /// What a bare instant selector over a metric looks back; `None` when /// the metric declares no sample interval. sample_interval_of: &'a dyn Fn(&str) -> Option, + /// The duration D of the query range, in ms (0 for an instant query): + /// `@ start()` is lowered as `@ end() offset D`. + query_range_ms: i64, } #[derive(Debug, Clone)] @@ -141,6 +144,7 @@ impl PromqlLowerer<'_> { accuracy: &AccuracyTarget, sample_interval_of: &dyn Fn(&str) -> Option, metric_type_of: &dyn Fn(&str) -> Option, + query_range_ms: i64, ) -> Result { let ast = parser::parse(query).map_err(LoweringError::Parse)?; check_depth(&ast, MAX_DEPTH)?; @@ -149,6 +153,7 @@ impl PromqlLowerer<'_> { let lowerer = PromqlLowerer { accuracy: accuracy.clone(), sample_interval_of, + query_range_ms, }; if metrics .iter() @@ -240,7 +245,7 @@ impl PromqlLowerer<'_> { child: Rc::new(self.walk(&sq.expr)?), }; // `offset`/`@` move the whole subquery, including its step grid. - let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; + let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref(), self.query_range_ms)?; Ok(if shift.is_identity() { subquery } else { @@ -251,11 +256,11 @@ impl PromqlLowerer<'_> { }) } Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; + let (metric, matchers, shift) = vs_parts(vs, self.query_range_ms)?; self.instant_source(metric, matchers, shift) } Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; + let (metric, matchers, shift) = vs_parts(&ms.vs, self.query_range_ms)?; Ok(Unresolved::TimeRange { range: ms.range, kind: TimeRangeKind::Range, @@ -993,7 +998,7 @@ impl PromqlLowerer<'_> { fn lower_inner(&self, expr: &Expr) -> Result { match expr { Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; + let (metric, matchers, shift) = vs_parts(vs, self.query_range_ms)?; Ok(Inner { metric, matchers, @@ -1003,7 +1008,7 @@ impl PromqlLowerer<'_> { }) } Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; + let (metric, matchers, shift) = vs_parts(&ms.vs, self.query_range_ms)?; Ok(Inner { metric, matchers, @@ -1026,7 +1031,8 @@ impl PromqlLowerer<'_> { call.func.name.to_string(), )); }; - let (metric, matchers, window, shift) = extract_matrix(arg(call, matrix_idx)?)?; + let (metric, matchers, window, shift) = + extract_matrix(arg(call, matrix_idx)?, self.query_range_ms)?; Ok(Inner { metric, matchers, @@ -1577,12 +1583,12 @@ fn clamp_bounds_filter(min: &Scalar, max: &Scalar, child: Unresolved) -> Unresol fn collect_metric_names(expr: &Expr, out: &mut Vec) { match expr { Expr::VectorSelector(vs) => { - if let Ok((metric, ..)) = vs_parts(vs) { + if let Ok((metric, ..)) = vs_parts(vs, 0) { out.push(metric); } } Expr::MatrixSelector(ms) => { - if let Ok((metric, ..)) = vs_parts(&ms.vs) { + if let Ok((metric, ..)) = vs_parts(&ms.vs, 0) { out.push(metric); } } @@ -1714,7 +1720,7 @@ fn resolve_group(agg: &AggregateExpr) -> Result<(Vec, bool)> { // ── Free helpers ────────────────────────────────────────────────────────────── -fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec, TimeShift)> { +fn vs_parts(vs: &VectorSelector, query_range_ms: i64) -> Result<(String, Vec, TimeShift)> { // A non-equality `__name__` matcher (`=~` / `!~` / `!=`) selects *across* // metric names. `Source::TimeSeries { metric }` carries a single concrete // metric name, so there is no representation for a regex/negated name @@ -1761,26 +1767,45 @@ fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec, TimeShift)> { .collect(); ms.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.value.cmp(&b.value))); let matchers = ms.into_iter().map(matcher_to_compare).collect(); - let shift = time_shift(vs.offset.as_ref(), vs.at.as_ref())?; + let shift = time_shift(vs.offset.as_ref(), vs.at.as_ref(), query_range_ms)?; Ok((metric, matchers, shift)) } /// Convert the parser's `offset` / `@` modifiers into a [`TimeShift`] (issue /// #40). Offset is signed milliseconds; `@ ` (parser seconds → ms) becomes -/// an absolute anchor, `@ start()`/`@ end()` the range bounds. -fn time_shift(offset: Option<&Offset>, at: Option<&ParserAtModifier>) -> Result { +/// an absolute anchor and `@ end()` the end of the query range. `@ start()` +/// is `@ end()` with `query_range_ms` (D) more offset: the range starts D +/// before it ends. +fn time_shift( + offset: Option<&Offset>, + at: Option<&ParserAtModifier>, + query_range_ms: i64, +) -> Result { let offset_ms = match offset { None => 0, Some(Offset::Pos(d)) => duration_ms(*d)?, Some(Offset::Neg(d)) => -duration_ms(*d)?, }; - let at = match at { - None => None, - Some(ParserAtModifier::Start) => Some(AtModifier::Start), - Some(ParserAtModifier::End) => Some(AtModifier::End), - Some(ParserAtModifier::At(t)) => Some(AtModifier::Timestamp(system_time_ms(*t)?)), - }; - Ok(TimeShift { offset_ms, at }) + Ok(match at { + None => TimeShift { + offset_ms, + at: None, + }, + Some(ParserAtModifier::End) => TimeShift { + offset_ms, + at: Some(AtModifier::End), + }, + Some(ParserAtModifier::Start) => TimeShift { + offset_ms: offset_ms.checked_add(query_range_ms).ok_or_else(|| { + LoweringError::InvalidParameter("offset plus query range overflows i64 ms".into()) + })?, + at: Some(AtModifier::End), + }, + Some(ParserAtModifier::At(t)) => TimeShift { + offset_ms, + at: Some(AtModifier::Timestamp(system_time_ms(*t)?)), + }, + }) } /// A `Duration` as `i64` milliseconds, rejecting an overflow rather than @@ -1821,13 +1846,16 @@ fn matcher_to_compare(m: &Matcher) -> Scalar { } } -fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration, TimeShift)> { +fn extract_matrix( + expr: &Expr, + query_range_ms: i64, +) -> Result<(String, Vec, Duration, TimeShift)> { match expr { Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; + let (metric, matchers, shift) = vs_parts(&ms.vs, query_range_ms)?; Ok((metric, matchers, ms.range, shift)) } - Expr::Paren(p) => extract_matrix(&p.expr), + Expr::Paren(p) => extract_matrix(&p.expr, query_range_ms), // A range-vector function argument must be a (parenthesised) matrix // selector. Do NOT descend through an arbitrary `Call` — that would // silently strip an unsupported wrapper (`rate(deriv(m[5m]))` lowering diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index e661ee984..3606a7b96 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -1074,7 +1074,9 @@ fn at_modifier_lowers_to_a_time_shift() { let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { panic!("expected a TimeShift for `@ start()`"); }; - assert_eq!(shift.at, Some(AtModifier::Start)); + // An instant query's range starts where it ends: `@ end()`. + assert_eq!(shift.at, Some(AtModifier::End)); + assert_eq!(shift.offset_ms, 0); // Offset and `@` compose: `@ end() offset 5m` carries both. let qe = ok("http_requests_total @ end() offset 5m"); diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 7e706c414..4e51a9dc9 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -1,5 +1,6 @@ use std::rc::Rc; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; use datafusion::logical_expr::utils::{split_binary, split_conjunction}; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; @@ -137,17 +138,27 @@ impl SqlLowerer<'_> { Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), - // Fold a numeric cast of a literal, as DataFusion's constant folding - // does: type coercion turns `latency < 100` on a DOUBLE column into - // `latency < CAST(100 AS DOUBLE)`, which lowers as `latency < 100.0`. + // Fold a numeric or timestamp cast of a literal, as DataFusion's + // constant folding does: type coercion turns `latency < 100` on a + // DOUBLE column into `latency < CAST(100 AS DOUBLE)`, which lowers + // as `latency < 100.0`, and `ts >= '2026-11-01'` into a cast of the + // string, which lowers as a `Timestamp` literal (ms since the epoch). Expr::Cast(c) if matches!(c.expr.as_ref(), Expr::Literal(v, _) if !v.is_null()) - && (c.field.data_type().is_integer() || c.field.data_type().is_floating()) => + && (c.field.data_type().is_integer() + || c.field.data_type().is_floating() + || matches!(c.field.data_type(), ArrowDataType::Timestamp(..))) => { let Expr::Literal(sv, _) = c.expr.as_ref() else { unreachable!() }; - match sv.cast_to(c.field.data_type()) { + let to = match c.field.data_type() { + ArrowDataType::Timestamp(..) => { + ArrowDataType::Timestamp(TimeUnit::Millisecond, None) + } + other => other.clone(), + }; + match sv.cast_to(&to) { Ok(folded) => scalar_value_to_asap(&folded).map(Unresolved::Literal), Err(_) => Ok(Unresolved::Cast { expr: bx(&c.expr)?, diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs index cf18399ad..08cb21fa9 100644 --- a/crates/frontend-sql/src/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -53,6 +53,13 @@ pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result Ok(ScalarValue::Utf8(s.clone())), DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), + // A timestamp is ms since the epoch, as the executor's columns. + DfScalarValue::TimestampSecond(Some(s), _) => Ok(ScalarValue::Timestamp(s * 1_000)), + DfScalarValue::TimestampMillisecond(Some(ms), _) => Ok(ScalarValue::Timestamp(*ms)), + DfScalarValue::TimestampMicrosecond(Some(us), _) => Ok(ScalarValue::Timestamp(us / 1_000)), + DfScalarValue::TimestampNanosecond(Some(ns), _) => { + Ok(ScalarValue::Timestamp(ns / 1_000_000)) + } // All three of DataFusion's interval scalars land on one canonical // shape; the narrower two simply leave the fields they do not carry // at zero. diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index fcb2b731d..1a1e58640 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -3284,3 +3284,35 @@ async fn trailing_window_on_an_aliased_table_and_under_asap_rate() { let (window, _) = trailing_window(below).expect("the WHERE window below it"); assert_eq!(window.as_millis(), 300_000); } + +// An absolute timestamp filter compares the time column with a `Timestamp` +// literal (ms since the epoch), so summary coverage lifts it into an +// interval on the timestamp column: the cast of the string folds, and the +// cast of the Timestamp column to DataFusion's nanosecond unit is dropped. +#[tokio::test] +async fn an_absolute_timestamp_filter_compares_the_column_with_a_timestamp_literal() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + for sql in [ + "SELECT COUNT(*) FROM metrics WHERE ts >= '2026-11-01'", + "SELECT COUNT(*) FROM metrics WHERE ts >= TIMESTAMP '2026-11-01 00:00:00'", + ] { + let qe = lower(sql).await; + let scan = OperatorNode::reachable(&qe) + .into_iter() + .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Scan { .. }))) + .unwrap_or_else(|| panic!("no Scan for {sql}")); + let NonASAPOp::Scan { predicates, .. } = op(&scan) else { + unreachable!() + }; + assert_eq!( + predicates.iter().map(|pred| &pred.0).collect::>(), + [&ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Ge, + right: Box::new(ScalarExpr::Literal(ScalarValue::Timestamp(NOV_1_2026_MS))), + semantics: ExprSemantics::Sql, + }], + "{sql}" + ); + } +} diff --git a/crates/types/src/ir/operator/operator_properties.rs b/crates/types/src/ir/operator/operator_properties.rs index e2a6ef1da..22181e69b 100644 --- a/crates/types/src/ir/operator/operator_properties.rs +++ b/crates/types/src/ir/operator/operator_properties.rs @@ -372,8 +372,6 @@ pub struct VectorMatch { /// of the query evaluation time (issue #40). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum AtModifier { - /// `@ start()` — the query range's start instant. - Start, /// `@ end()` — the query range's end instant. End, /// `@ ` — an absolute instant, milliseconds since the Unix epoch (may be diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 8a9e5893f..f035dbda5 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,15 +3,17 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived once for each summary-state memo group, from its -//! input groups, never declared, and stored in its `GroupProperties`. -//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! Coverage is derived once for each aggregate memo group (a summary state +//! or an exact `Aggregate`), from its input groups, never declared, and +//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a +//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a //! predicate conjunct moves into `selection` when it can be lifted to the -//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without -//! `@` and direct-column `Project` items) and it is a value set or an -//! interval on one column. Everything else stays in `definition` as a -//! residual, so two states are merged only when they compute the same thing -//! over disjoint rows. Node trees carry no coverage. +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; @@ -23,7 +25,7 @@ use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::operator::operator_properties::Reduction; +use crate::ir::operator::operator_properties::{AtModifier, Reduction}; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; @@ -234,12 +236,12 @@ mod limit_bound { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum CoverageError { - /// The node is not a summary state (`SummaryAgg`, `SummaryTumblingWindowChain` or - /// `SummaryMerge`): a caller bug, or bad input when it is a merge's - /// input. - #[error( - "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" - )] + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] NotSummary, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] @@ -280,10 +282,15 @@ pub enum CoverageError { type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { - /// Coverage of a summary state a memo group computes with `operator` - /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// Coverage of what a memo group computes with `operator` over its + /// input groups: a `SummaryAgg`, an exact `Aggregate`, a + /// `SummaryTumblingWindowChain` (a build over its input, the timestamp + /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the + /// chain's coverage with the windows' interval on the timestamp column), /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails + /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` + /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge + /// reads its inputs' stored coverage and fails /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may @@ -296,14 +303,43 @@ impl SummaryCoverage { op.map_children(|input| InputOrTemporaryBuild::Input(*input)) }; match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = + unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } Operator::NonASAP(_) => match selected_build(operator)? { Some(build) => of_summary_agg(&build), None => Err(CoverageError::NotSummary), }, Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( - "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" - ), + Operator::ASAP(op @ ASAPOp::SummaryTumblingWindowChain { .. }) => { + let ASAPOp::SummaryTumblingWindowChain { + child, + family, + input, + reduction, + grouping, + filter, + .. + } = as_built(op) + else { + unreachable!("matched a chain"); + }; + of_summary_agg(&ASAPOp::SummaryAgg { + child, + family, + input, + reduction, + grouping, + filter, + }) + } Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -316,7 +352,7 @@ impl SummaryCoverage { .ok_or(CoverageError::NotSummary) }) .collect::, _>>()?; - let merged = of_merge(&inputs)?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; match group_by { None => Ok(merged), Some(keys) => regrouped(merged, &children[0], keys), @@ -409,8 +445,12 @@ fn selected_build<'m>( })) } -/// The coverage of a merge of states covering `inputs`. -fn of_merge(inputs: &[SummaryCoverage]) -> Result { +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[SummaryCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs .iter() @@ -427,12 +467,16 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result None, } .unwrap_or(MergeInputRule::Disjoint); - let separated: fn(&SelectionBox, &SelectionBox) -> bool = match required { - MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => todo!( - "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" - ), - MergeInputRule::Disjoint => SelectionBox::disjoint, + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), }; // Every pair of boxes of different inputs. if required != MergeInputRule::MayOverlap @@ -547,9 +591,15 @@ fn build_input_schema(state: &I) -> Option { } } +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + /// The coverage of `build`, a `SummaryAgg` over memo groups (or over an -/// input a rule built over them): its operators walked down through the -/// liftable operators of its input groups' expressions. +/// input a rule built over them). fn of_summary_agg(build: &ASAPOp>) -> Result { let ASAPOp::SummaryAgg { child, @@ -562,6 +612,27 @@ fn of_summary_agg(build: &ASAPOp>) -> Result, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result { // The operators a predicate can be lifted through, top to bottom, and // the first input below them. A subquery's group stays named in the // residual of the operator that reads it. @@ -572,7 +643,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result true, Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), + Operator::NonASAP(NonASAPOp::TimeShift { .. }) => true, _ => false, }; if !passes { @@ -589,9 +660,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>>> = chain .iter() .enumerate() @@ -625,14 +694,26 @@ fn of_summary_agg(build: &ASAPOp>) -> Result None, }; - let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: agg_filter, - })]; + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + let combined = match lifted.get(&column) { + Some(existing) => existing.intersect(&constraint)?, + None => constraint, + }; + Some((column, combined)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; for (depth, link) in chain.iter().enumerate() { match link { Operator::NonASAP(NonASAPOp::Filter { .. }) => { @@ -643,9 +724,8 @@ fn of_summary_agg(build: &ASAPOp>) -> Result todo!( - "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" - ), + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} other => unlifted_operators.push(unlifted(other)), } } @@ -660,14 +740,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result>, top: &Schema, @@ -695,15 +767,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - let field = &top.fields[column]; - let namesakes = top - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - if namesakes != 1 { - return None; - } + column_identity(top, column)?; let combined = match lifted.get(&column) { Some(existing) => existing.intersect(&constraint)?, None => constraint, @@ -732,8 +796,8 @@ fn group_id(input: &GroupInput<'_>) -> Option { } } -/// `link`, a liftable operator kept whole in a definition: its main input -/// `None`, every other input the group it is. +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input the group it is. fn unlifted(link: &Operator>) -> Operator> { let mut operator = link.map_children(group_id); match &mut operator { @@ -741,9 +805,10 @@ fn unlifted(link: &Operator>) -> Operator> { NonASAPOp::Filter { child, .. } | NonASAPOp::Project { child, .. } | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. }, + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, ) => *child = None, - _ => unreachable!("only liftable operators are walked"), + _ => unreachable!("only liftable operators and aggregates are unlifted"), } operator } @@ -761,6 +826,114 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `[anchor − o − W, anchor − o)` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o. The anchor is the evaluation time +/// t (`EvaluationTimePlusMs`), also for `@ end()`, or T for `@ T` +/// (`FixedValue(Timestamp)`). `None` when they are not one interval: no or +/// several range `TimeRange`s, an instant one, or several `@`s. +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::End) | None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Included(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Excluded(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator>, +) -> Result { + let mut links = vec![selection.clone()]; + let mut below = *selection.children()[0]; + loop { + match below.operator() { + link @ Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => { + below = child; + links.push(link); + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => break, + _ => return Err(CoverageError::NotSummary), + } + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NotSummary)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = match columns.get(&time_column) { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint.clone()), + }; + columns.insert( + time_column.clone(), + combined.ok_or(CoverageError::NotSummary)?, + ); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(SummaryCoverage { + definition: chain.definition.clone(), + selection, + }) +} + /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index bb9413413..f288a4b02 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -313,7 +313,7 @@ fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { /// Whether `left` and `right` are one candidate of a group: the same /// expression ([`same_expression`]) and logical properties (parameters by -/// [`same_value`], window layout). +/// [`same_value`]). fn same_expression_and_logical_properties( left: &CandidateForMemoGroup, right: &CandidateForMemoGroup, @@ -323,7 +323,6 @@ fn same_expression_and_logical_properties( &left.logical_properties.parameters, &right.logical_properties.parameters, ) - && left.logical_properties.window_layout == right.logical_properties.window_layout } /// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. @@ -1352,7 +1351,9 @@ mod memo_tests { use super::*; use crate::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::{DataType, Field, Schema, SketchStatistic, SummaryUpdate}; + use crate::ir::schema::{ + DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchStatistic, SummaryUpdate, + }; use crate::ir::NonASAPOp; /// `Scan metric[1m]`: `[ts, value]`. @@ -1416,7 +1417,7 @@ mod memo_tests { }) } - /// A candidate computing `operator` with `parameters` and no window. + /// A candidate computing `operator` with `parameters`. fn candidate( operator: Operator, parameters: Option, @@ -1425,10 +1426,7 @@ mod memo_tests { CandidateForMemoGroup { // Built directly: the doc example's k = 200 and k = 345 are not // in today's KLL list, which `LogicalProperties::new` checks. - logical_properties: LogicalProperties { - parameters, - window_layout: None, - }, + logical_properties: LogicalProperties { parameters }, expression: CandidateExpression { operator }, input_requirements, fused_groups: vec![], @@ -1539,10 +1537,8 @@ mod memo_tests { index_in_group: 0 } ); - let mut windowed = candidate(kll_state(g1), kll_k(200)); - windowed.logical_properties.window_layout = - Some(WindowLayout::Tumbling { width_ms: 10_000 }); - assert_ne!(memo.add_candidate(g2, windowed).unwrap(), first); + let other_size = candidate(kll_state(g1), kll_k(345)); + assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); } diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 6678a0020..0d3fceed8 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -45,10 +45,9 @@ pub struct MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, - /// What an aggregate group covers: a summary state, a chain, a merge - /// over a chain's windows, or an exact `Aggregate` result; `None` for - /// any other group. Derived today for summary states only; exact - /// `Aggregate` groups follow in #1207 PR 2. + /// What an aggregate group covers: a summary state, a chain, a + /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact + /// `Aggregate` result; `None` for any other group. pub aggregate_coverage: Option, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs index cf96a6c45..5cb29b19a 100644 --- a/crates/types/tests/memo_summary_coverage.rs +++ b/crates/types/tests/memo_summary_coverage.rs @@ -1,5 +1,5 @@ -//! Group properties and summary coverage, derived once per memo group from -//! its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` +//! Group properties and aggregate coverage, derived once per memo group +//! from its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` //! §4.2.2). The memo of the worked example: //! //! ```text @@ -11,27 +11,35 @@ use std::ops::Bound; use std::time::Duration; -use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; +use asap_types::ir::operator::operator_properties::{ + AtModifier, GroupKeys, Reduction, Source, TimeShift, +}; +use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, CoverageError, SummaryCoverage, + ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, SummaryCoverage, }; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, MergeInputRule, Schema, SketchAlgorithm, - SketchStatistic, SummaryUpdate, + DataType, ExactKind, Field, FieldDataType, GroupingStrategy, MergeInputRule, Schema, + SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, SchemaDerivationError, TimeRangeKind, }; -use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, + MemoGroupId, OriginOfCandidateForMemoGroup, +}; use asap_types::test_support::definition_id; +const TS: usize = 0; const JOB: usize = 2; const A: usize = 3; const B: usize = 4; const MINUTE_MS: i64 = 60_000; +const HOUR_MS: i64 = 60 * MINUTE_MS; /// `Scan latency{predicates}`: `[ts, value, job, a, b]`. fn scan(predicates: Vec>) -> Operator { @@ -85,6 +93,14 @@ fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + fn range(width_ms: i64, child: MemoGroupId) -> Operator { Operator::NonASAP(NonASAPOp::TimeRange { range: Duration::from_millis(width_ms as u64), @@ -93,22 +109,43 @@ fn range(width_ms: i64, child: MemoGroupId) -> Operator { }) } +/// `SummaryAgg[family]` of the sample value per series. +fn state(family: FieldDataType, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + family, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }) +} + +fn kll_family() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) +} + /// `SummaryAgg[KLL]` of the sample value per series. fn kll(child: MemoGroupId) -> Operator { - Operator::ASAP(ASAPOp::SummaryAgg { + state(kll_family(), child) +} + +/// `SummaryTumblingWindowChain[KLL, width_ms]` of the sample value per series. +fn kll_chain(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child, - family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), + family: kll_family(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, grouping: GroupingStrategy::default(), filter: None, + tumbling_window_width_ms: width_ms as u64, }) } fn merge(children: Vec) -> Operator { Operator::ASAP(ASAPOp::SummaryMerge { children, - newest_tumbling_windows: None, group_by: None, }) } @@ -127,7 +164,7 @@ fn kll_of_last_minute_where( fn coverage(memo: &Memo, group: MemoGroupId) -> &SummaryCoverage { memo.groups()[group.0 as usize] .group_properties - .summary_coverage + .aggregate_coverage .as_ref() .expect("a summary state") } @@ -146,8 +183,29 @@ fn jobs(values: &[&str]) -> Constraint { ) } -fn last_minute() -> Option<(Bound, Bound)> { - Some((Bound::Excluded(-MINUTE_MS), Bound::Included(0))) +/// `[t + lower_ms, t + upper_ms)` on the timestamp column, t the evaluation +/// time. +fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Included(IntervalConstraintLimit::EvaluationTimePlusMs(lower_ms)), + upper: Bound::Excluded(IntervalConstraintLimit::EvaluationTimePlusMs(upper_ms)), + } +} + +fn last_minute() -> Constraint { + relative(-MINUTE_MS, 0) +} + +fn ts() -> ColumnIdentity { + ColumnIdentity::new(None, "ts") +} + +/// The constraint on the timestamp column of `group`'s one selection box. +fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { + let [selection] = coverage(memo, group).selection() else { + panic!("one box: {:?}", coverage(memo, group)) + }; + selection.columns()[&ts()].clone() } /// A group's properties come from its operator and its input groups' @@ -174,14 +232,14 @@ fn group_properties_come_from_input_groups() { .unwrap(); let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; assert_eq!(properties(g1).schema.fields.len(), 5); - assert_eq!(properties(g1).summary_coverage, None); + assert_eq!(properties(g1).aggregate_coverage, None); assert_eq!(properties(g4).result_kind, OperatorResultKind::State); assert!(properties(g4) .schema .fields .iter() .any(|field| field.name == "state" && !field.is_plain())); - assert!(properties(g4).summary_coverage.is_some()); + assert!(properties(g4).aggregate_coverage.is_some()); assert_eq!( properties(readout).result_kind, OperatorResultKind::InstantVector @@ -216,7 +274,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { selection.columns()[&ColumnIdentity::new(None, "job")], jobs(&["api"]) ); - assert_eq!(selection.relative_time(), last_minute()); + assert_eq!(selection.columns()[&ts()], last_minute()); } /// G7 has G4's definition and id and a disjoint selection, so @@ -378,29 +436,221 @@ fn scan_predicates_lift_and_their_states_merge() { ); } -/// A tumbling-window chain of five minutes covers what one build over the -/// five-minute range does: the same definition and selection. +/// A chain has the definition of a build over its input and leaves the +/// timestamp column unconstrained. `TimeRange` and `TimeShift` over it +/// select windows as over raw rows, so the merge over that selection covers +/// what the direct build does, and `add_candidate` accepts the merge in the +/// direct build's group: +/// +/// ```text +/// G1 Scan latency +/// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) +/// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) +/// G4 candidate: SummaryMerge(G7) +/// ``` #[test] -fn a_tumbling_window_chain_covers_its_whole_window() { +fn a_merge_over_a_chains_windows_covers_the_direct_build() { let mut memo = Memo::default(); let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); - let chain = memo - .find_or_add_group(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: g1, - family: FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms: MINUTE_MS as u64, - tumbling_window_count: 5, - })) + let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); + let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(kll(g3)).unwrap(); + let g5 = memo.find_or_add_group(kll_chain(MINUTE_MS, g1)).unwrap(); + let g6 = memo.find_or_add_group(shift(HOUR_MS, None, g5)).unwrap(); + let g7 = memo.find_or_add_group(range(5 * MINUTE_MS, g6)).unwrap(); + + let whole_chain = coverage(&memo, g5); + assert_eq!(whole_chain.definition(), coverage(&memo, g4).definition()); + assert!(whole_chain.selection()[0].columns().is_empty()); + let five_minutes_an_hour_ago = relative(-HOUR_MS - 5 * MINUTE_MS, -HOUR_MS); + assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); + assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); + + let expression = CandidateExpression { + operator: merge(vec![g7]), + }; + let merge_of_windows = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::new(None)], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(g4, merge_of_windows).unwrap(); + let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); + assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); +} + +/// The range and every offset lift into one interval relative to the +/// evaluation time t; `@ end()` is the same anchor as no `@`; `@ T` fixes +/// the interval in absolute time: +/// +/// | Group | Expression | Timestamp column | +/// |---|---|---| +/// | G2 | `TimeRange[5m](G1)` (`m[5m]`) | `[t − 5m, t)` | +/// | G4 | `TimeRange[5m](TimeShift(1h)(G1))` (`m[5m] offset 1h`) | `[t − 65m, t − 60m)` | +/// | G6 | `TimeRange[5m](TimeShift(@ end())(G1))` | `[t − 5m, t)` | +/// | G8 | `TimeRange[5m](TimeShift(@ 1700000000)(G1))` | `[1699999700000, 1700000000000)` | +/// +/// each under a `SummaryAgg[KLL]`. +#[test] +fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let kll_over = |memo: &mut Memo, expression: Operator| { + let shifted = memo.find_or_add_group(expression).unwrap(); + let ranged = memo + .find_or_add_group(range(5 * MINUTE_MS, shifted)) + .unwrap(); + memo.find_or_add_group(kll(ranged)).unwrap() + }; + let plain = { + let ranged = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + memo.find_or_add_group(kll(ranged)).unwrap() + }; + let offset = kll_over(&mut memo, shift(HOUR_MS, None, g1)); + let at_end = kll_over(&mut memo, shift(0, Some(AtModifier::End), g1)); + let at_t = kll_over( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_000_000)), g1), + ); + assert_eq!(time_of(&memo, plain), relative(-5 * MINUTE_MS, 0)); + assert_eq!(time_of(&memo, offset), relative(-3_900_000, -3_600_000)); + assert_eq!(coverage(&memo, at_end), coverage(&memo, plain)); + let timestamp = |ms| IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(ms)); + assert_eq!( + time_of(&memo, at_t), + Constraint::Interval { + lower: Bound::Included(timestamp(1_699_999_700_000)), + upper: Bound::Excluded(timestamp(1_700_000_000_000)), + } + ); + for state in [offset, at_end, at_t] { + assert_eq!( + coverage(&memo, state).definition(), + coverage(&memo, plain).definition() + ); + } +} + +/// `ts >= ` (a SQL timestamp filter) and `value > 10` lift as +/// intervals between fixed values: `G3 SummaryAgg[KLL](G2)`, `G2 +/// Filter(ts >= 2026-11-01 ∧ value > 10)(G1)`. +#[test] +fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + let compare = |column, op, value| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Sql, + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo + .find_or_add_group(filter( + ScalarExpr::BoolAnd(vec![ + compare(TS, CompareOpKind::Ge, ScalarValue::Timestamp(NOV_1_2026_MS)), + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ]), + g1, + )) + .unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + let fixed = |value| Bound::Included(IntervalConstraintLimit::FixedValue(value)); + assert_eq!( + time_of(&memo, g3), + Constraint::Interval { + lower: fixed(ScalarValue::Timestamp(NOV_1_2026_MS)), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( + 10.0 + ))), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).definition().unlifted_operators, + vec![kll_unlifted()] + ); +} + +/// Counter states (`Increase`) merge only over windows disjoint in time: +/// the last 5 minutes and the 5 before them merge into the last 10; two +/// copies of the last 5 minutes do not. A KLL over the last minute and one +/// over the minute before it are disjoint too. +/// +/// ```text +/// G2 TimeRange[5m](G1); G3 SummaryAgg[Increase](G2) +/// G4 TimeShift(5m)(G1); G5 TimeRange[5m](G4); G6 SummaryAgg[Increase](G5) +/// G7 SummaryMerge(G3, G6) +/// ``` +#[test] +fn counter_states_merge_over_windows_disjoint_in_time() { + let increase = || FieldDataType::exact(ExactKind::Increase); + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); + let g4 = memo + .find_or_add_group(shift(5 * MINUTE_MS, None, g1)) .unwrap(); - let five_minutes = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); - let direct = memo.find_or_add_group(kll(five_minutes)).unwrap(); - assert_eq!(coverage(&memo, chain), coverage(&memo, direct)); + let g5 = memo.find_or_add_group(range(5 * MINUTE_MS, g4)).unwrap(); + let g6 = memo.find_or_add_group(state(increase(), g5)).unwrap(); + let g7 = memo.find_or_add_group(merge(vec![g3, g6])).unwrap(); + assert_eq!(time_of(&memo, g7), relative(-10 * MINUTE_MS, 0)); + assert!(matches!( + memo.find_or_add_group(merge(vec![g3, g3])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::DisjointInTime + } + )) + )); + let g8 = memo.find_or_add_group(kll(g2)).unwrap(); + let g9 = memo.find_or_add_group(kll(g5)).unwrap(); + let g10 = memo.find_or_add_group(merge(vec![g8, g9])).unwrap(); + assert_eq!(time_of(&memo, g10), relative(-10 * MINUTE_MS, 0)); +} + +/// An exact `Aggregate` gets coverage like a summary state: `G4 +/// Aggregate[sum(value) by job](G3)` over the last minute of `job = "api"` +/// has the `Aggregate` as its definition and the job and minute as its +/// selection. +#[test] +fn an_exact_aggregate_has_coverage() { + let sum_by_job = |child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![JOB]), + measures: vec![AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); + let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); assert_eq!( - coverage(&memo, chain).selection()[0].relative_time(), - Some((Bound::Excluded(-5 * MINUTE_MS), Bound::Included(0))) + coverage.definition().unlifted_operators, + vec![sum_by_job(MemoGroupId(0)).map_children(|_| None)] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) ); + assert_eq!(selection.columns()[&ts()], last_minute()); } diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 19920823c..fe98c2b56 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -115,7 +115,7 @@ A node in the physical data will represent the data or summary instance, so a no | How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | | How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `GroupProperties::summary_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -235,14 +235,14 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates a summary-state group, `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::summary_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint. +Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). ```rust /// What a summary state computes, with every lifted selection removed. pub struct SummaryStateDefinition { /// The group the unlifted operators read: the first input under the - /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ - /// / direct-column Project. `None` when that input is a `Scan` with a + /// SummaryAgg or Aggregate that is not Filter / range TimeRange / + /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a /// liftable predicate (a PromQL label matcher): it is lifted too, and the /// `Scan` with the rest of its predicates is the last unlifted operator. pub input_group: Option, @@ -283,31 +283,31 @@ G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryA | Step | Group | Expression | What the walk does | |---|---|---|---| | 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | -| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as time `(−60 s, 0]` | +| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as `ts ∈ [t − 60 s, t)` (t the evaluation time) | | 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | | 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | ```text -G4.summary_coverage = SummaryCoverage { +G4.aggregate_coverage = SummaryCoverage { definition: SummaryStateDefinition { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL]], }, - selection: [ { job ∈ {api}, time (−60 s, 0] } ], + selection: [ { job ∈ {api}, ts ∈ [t − 60 s, t) } ], } ``` -**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, time (−60 s, 0] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, time (−60 s, 0] }`. +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, ts ∈ [t − 60 s, t) }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ [t − 60 s, t) }`. **Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: ```text -G10.summary_coverage = SummaryCoverage { +G10.aggregate_coverage = SummaryCoverage { definition: SummaryStateDefinition { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, - selection: [ { job ∈ {api}, time (−60 s, 0] } ], + selection: [ { job ∈ {api}, ts ∈ [t − 60 s, t) } ], } ``` @@ -329,13 +329,23 @@ The `definition` is what remains of the operators the walk passed, with every co | a `Filter` whose conditions all move into `selection` | removed | | a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | | `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | -| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | +| the one range `TimeRange`, and every `TimeShift` (with or without `@`), when they become one interval on the timestamp column | removed | +| a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | | a direct-column `Project` | unchanged | | a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | | any other operator (the first group the walk cannot pass) | not in the list: it is the `input_group` | -A `SummaryTumblingWindowChain` covers what one build over its whole window does: the walk starts at that build's `SummaryAgg` over a range `TimeRange` of the whole window over the chain's input. A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. +An exact `Aggregate` is the first unlifted operator, kept whole (its per-measure filters and `HAVING` read its input or output, not the rows it reads). A `SummaryTumblingWindowChain[w](X)` has the definition of a build over X, and its selection leaves the timestamp column unconstrained: a chain has no range. A `TimeRange` or `TimeShift` over a chain (a group whose output is the chain's state with a time index) has the chain's coverage with the interval they select on the timestamp column of X, computed as over raw rows. A `SummaryMerge` over that group reads it, so `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` covers exactly what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does, and `Memo::add_candidate` accepts the merge as a candidate of the direct build's group: + +```text +G1 Scan latency +G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) ts ∈ [t − 65m, t − 60m) +G5 Chain[KLL, 1m](G1) ts unconstrained +G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) ts ∈ [t − 65m, t − 60m) +G4 candidate: SummaryMerge(G7) the same coverage as G4 +``` + +A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. **When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input_group` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. @@ -379,11 +389,11 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl | Operator it must pass | Can it pass? | Why | Example | |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | -| `TimeRange` (range) or `TimeShift` (without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | +| `TimeRange` (range) or `TimeShift` (with or without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | | `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | -| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit`, a `TimeShift` with `@` | no | | | +| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: @@ -423,18 +433,22 @@ Any other shape stays in `definition`: - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. -**Time.** Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`. Its KLL state (one per series) has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = time `(−3m, −2m]`, the minute that ended 2 minutes before evaluation. There are two kinds of time: +**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ [anchor − o − W, anchor − o)` on the timestamp column of the `TimeRange`'s input. The anchor is t, also for `@ end()`, or T for `@ T`. The PromQL front end lowers `@ start()` as `@ end()` with D more offset, D the query range's duration (0 for an instant query). Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ [t − 3m, t − 2m)`. -| Kind | Comes from | Example | +| Query fragment | `lower` | `upper` | |---|---|---| -| **Relative** to the evaluation time | a range `TimeRange(w)` over a `TimeShift(s)` → `(−(s+w), −s]`; a SQL trailing window `ts >= now() - INTERVAL 'w'` lowers to `TimeRange(w)` | `TimeRange(1m)` alone → `(−1m, 0]`; over `TimeShift(1m)` → `(−2m, −1m]` | -| **Absolute** (later) | an interval on the timestamp column | `ts >= t0 AND ts < t1` → `ts ∈ [t0, t1)` | +| PromQL `m[5m]` | `Included(EvaluationTimePlusMs(-300_000))` | `Excluded(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Included(EvaluationTimePlusMs(-3_900_000))` | `Excluded(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ end()` | as `m[5m]` | | +| PromQL `m[5m] @ start()`, query range D | `Included(EvaluationTimePlusMs(-D - 300_000))` | `Excluded(EvaluationTimePlusMs(-D))` | +| PromQL `m[5m] @ 1700000000` | `Included(FixedValue(Timestamp(1_699_999_700_000)))` | `Excluded(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= now() - INTERVAL '5 minutes'` | lowers to `TimeRange(5m)`: as `m[5m]` | | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(1_793_491_200_000)))` | `Unbounded` | -- PromQL windows exclude their start, so relative windows are open on the left. A SQL trailing window takes the same timing, `>=` included. -- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) builds its tumbling windows this way: a 3-minute window as tumbling windows `(−1m, 0]`, `(−2m, −1m]`, `(−3m, −2m]`, so tumbling window times are derived, not declared (#601). +- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) reads a longer range from a chain: `TimeRange(3m)` over a 1-minute chain selects the windows `[t − 1m, t)`, `[t − 2m, t − 1m)`, `[t − 3m, t − 2m)`, and the merge over them covers `[t − 3m, t)`. +- Counter states (`Rate`, `Increase`) merge only when their intervals on the timestamp column are disjoint (`MergeInputRule::DisjointInTime`); other families need any column disjoint. - An instant `TimeRange` (latest sample per series) does not pick rows by time, so it stays in `definition`. The walk also stops there, so for now the label matchers of an instant selector stay in `definition` too; lifting them is planned together with `rate` (Rule 1). -- The IR cannot yet write a timestamp constant, so absolute SQL time filters stay in `definition` for now. -- Absolute and relative time are never compared: a state over `(−1m, 0]` and one over `ts ∈ [t0, t1)` are treated as possibly overlapping. +- Limits of different variants never compare: a state over `ts ∈ [t − 1m, t)` and one over `ts ∈ [t0, t1)` are treated as possibly overlapping, and a relative interval and an absolute timestamp condition on one path do not combine (time then stays in `definition`). **More examples** of what goes where. Each query's quantile is answered from a KLL `SummaryAgg`: @@ -442,7 +456,7 @@ Any other shape stays in `definition`: |---|---|---|---| | SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | | SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | the last 3 to 2 minutes before evaluation, `(−3m, −2m]` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ [t − 3m, t − 2m)` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | | PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | @@ -451,7 +465,7 @@ Any other shape stays in `definition`: #### 4.2.3 Cost of deriving coverage -**When it runs.** `find_or_add_group` derives a summary-state group's coverage once, when it creates the group, and stores it in `GroupProperties::summary_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. +**When it runs.** `find_or_add_group` derives an aggregate group's coverage once, when it creates the group, and stores it in `GroupProperties::aggregate_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. **Sizes used below.** @@ -734,7 +748,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ State: series labels, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: time (−2m, 0] + ▲ selection: ts ∈ [t − 2m, t) │ ┌─────────────────┴─────────────────┐ │ State: series labels, │ State: series labels, @@ -742,7 +756,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ │ [[ SummaryAgg ]] tumbling window 0 [[ SummaryAgg ]] tumbling window 1 KLL k=200, per series KLL k=200, per series - selection: time (−1m, 0] selection: time (−2m, −1m] + selection: ts ∈ [t − 1m, t) selection: ts ∈ [t − 2m, t − 1m) ▲ ▲ │ │ [ TimeRange 1m ] [ TimeRange 1m ] @@ -844,26 +858,26 @@ input groups output groups | State | Query | Selection | |---|---|---| -| `A` | `quantile_over_time(0.99, m[1m])` | time `(−1m, 0]` | -| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | time `(−2m, −1m]` | -| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | time `(−90s, −30s]` | -| `D` | `quantile_over_time(0.99, n[1m])` | time `(−1m, 0]`, but over metric `n`: definition `KLL(value) per series over Scan n` | +| `A` | `quantile_over_time(0.99, m[1m])` | ts ∈ `[t − 1m, t)` | +| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | ts ∈ `[t − 2m, t − 1m)` | +| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | ts ∈ `[t − 90s, t − 30s)` | +| `D` | `quantile_over_time(0.99, n[1m])` | ts ∈ `[t − 1m, t)`, but over metric `n`: definition `KLL(value) per series over Scan n` | | Merge | Allowed? | Why | Result's selection | |---|---|---|---| -| `A + B` | ✓ | same definition, no overlap | `(−2m, 0]` (touching time windows join) | -| `A + C` | ✗ | `(−60s, −30s]` is in both, so those rows would be counted twice | | +| `A + B` | ✓ | same definition, no overlap | `[t − 2m, t)` (touching time windows join) | +| `A + C` | ✗ | `[t − 60s, t − 30s)` is in both, so those rows would be counted twice | | | `A + A` | ✗ | every row is in both | | | `A + D` | ✗ | different definitions: `D` reads metric `n`, not `m` | | | `A` + a KLL with `k = 400` | ✗ | different schema: `k` is part of the state type, so the merge is rejected before coverage is checked | | -| `(A + B)` + the state of `m[1m] offset 2m`, time `(−3m, −2m]` | ✓ | a merge has coverage like any state, so merges nest | `(−3m, 0]` | +| `(A + B)` + the state of `m[1m] offset 2m`, ts ∈ `[t − 3m, t − 2m)` | ✓ | a merge has coverage like any state, so merges nest | `[t − 3m, t)` | **Whether inputs may overlap** depends on the summary type (family and algorithm, §3). This is #592 (open), which adds `FieldDataType::family_merges` and `merge_relation`; #646 alone requires every merge to be overlap-free: | Rule | Summary types | Example | |---|---|---| -| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(−60s, −30s]` would be counted twice | -| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(−1m, 0]` and `(−90s, −30s]`, which together cover `(−90s, 0]` | +| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `[t − 60s, t − 30s)` would be counted twice | +| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `[t − 1m, t)` and `[t − 90s, t − 30s)`, which together cover `[t − 90s, t)` | | **cannot merge (yet)** | `CmsWithHeap`, `CountSketchWithHeap`, Theta, KMV, exact `Rate`/`IRate`/`Increase`, samples, wavelets, models | no sound merge is modeled yet, so #592 rejects them | | **right inside left** | subtraction | 5.7 | @@ -871,7 +885,7 @@ input groups output groups |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | | When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | -| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | +| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`[t − 2m, t − 1m)` and `[t − 1m, t)` give `[t − 2m, t)`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | **Guarantee after a rollup.** The groups of one build partition its rows, so the states a rollup merges into one output group are disjoint, and the family-capability table's merged error applies. A rolled-up state of these families keeps the guarantee one build of the same size would have over the coarse group's rows; most are exactly that build's state: @@ -893,7 +907,7 @@ Rejected: counter states (`Rate`, `Increase`), which merge consecutive windows o **Other uses of coverage.** The planner also uses coverage to read or reuse a state without merging: - **Slice:** from a `by[region, job]` state, a query for `region = 'us'` by job reads only the `us` groups ✓. A query for `value < 50` ✗: `value` is not a grouping column, and a sketch cannot be filtered after it is built. -- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(−5m, 0]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. +- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `[t − 5m, t)` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. ### 5.7 Reserved operators (not implemented) @@ -936,7 +950,7 @@ OperatorNode §6.1 │ └── algorithm, params §6.4 └── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -MemoGroup.group_properties.summary_coverage: SummaryCoverage §6.6 +MemoGroup.group_properties.aggregate_coverage: SummaryCoverage §6.6 └── definition: SummaryStateDefinition { input_group, unlifted_operators } ``` @@ -947,7 +961,7 @@ MemoGroup.group_properties.summary_coverage: SummaryCoverage §6.6 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.family` | | §6.4 Summary-family parameters | `SketchKind`, `SketchParams`, `GroupingStrategy`, exact/sample/wavelet/model params | the payload of each non-plain `FieldDataType` variant | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::summary_coverage` of summary-state memo groups | +| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `family = Sketch(SketchKind::new(Kll, Kll { k: 200 }), PerSubpopulationInstance)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5). Its `schema` (§6.3) is `(job Utf8, state Sketch(KLL k=200))`: the `state` field has the same type as `family`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). From 30670f839730aa52946e05cb291374270dcbe47c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:08:55 +0000 Subject: [PATCH 4/4] test(sql): now() needs no coercion cast to a timestamp (#1207 PR 2) --- crates/frontend-sql/tests/sql_lowering.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 1a1e58640..0d9550b31 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2237,7 +2237,9 @@ async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { /// PromQL's Float64 Unix-seconds `EvalTimestamp`. #[tokio::test] async fn now_in_predicate_lowers_to_current_timestamp() { - // WHERE folds onto Scan.predicates (no explicit Filter node). + // WHERE folds onto Scan.predicates (no explicit Filter node). The + // coercion cast of `now()` to the column's timestamp type is dropped: + // `now()` is already a Timestamp. let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; let NonASAPOp::Project { child, .. } = op(&qe) else { panic!("expected Project at root, got {qe:?}"); @@ -2248,7 +2250,7 @@ async fn now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "NOW() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -2286,7 +2288,7 @@ async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "now() must lower to CurrentTimestamp, got {:?}", predicates[0].0 );