diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 940cd9c02..666477f7e 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -3,11 +3,11 @@ name = "asap-accuracy" version = "0.1.0" edition = "2021" -# The accuracy model shared by the planning stages: per-family estimators, -# guarantee composition and propagation through a DAG, sizing, budget -# split, the target check and the -# deployment's accuracy model and evidence traits. Depends only on -# `asap-types`, never on a stage crate, so every stage can use it. +# The accuracy rules shared by the planning stages: per-algorithm +# guarantees (the theoretical guarantee of θ), the target check, the +# deployment's measured accuracy, each operator's error contract and α^log +# of a logical edge. Depends only on `asap-types`, never on a stage crate, +# so every stage can use it. [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } diff --git a/crates/accuracy/src/budget.rs b/crates/accuracy/src/budget.rs deleted file mode 100644 index 65d552250..000000000 --- a/crates/accuracy/src/budget.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! Split an end-to-end accuracy target equally among the approximate inputs -//! of one composition (#669, part b). -//! -//! When `n` approximate values compose into one output, each is sized for an -//! equal share of the output's target, so the composed bound meets the -//! target: `ε_i = ε / n` for an additively composed metric (no slack), -//! `ε_i = 1 − (1 + ε)^{−1/n}` for a relative one, and `δ_i = δ / n` (the -//! union bound). Relative errors compose through products and quotients: -//! each input contributes a factor of at most `1 + ε_i` as a factor and -//! `1/(1 − ε_i)` as a divisor, so the true error of any mix stays within -//! `(1 − ε_i)^{−n} − 1 = ε`. The composed bound Stage 3 checks meets it for -//! one product or quotient of two inputs (a ratio's `(a + b)/(1 − b)`), not -//! for a nested quotient such as `x / (y / z)`, whose bound composes each -//! step's slack. A lone approximate denominator over an exact numerator is -//! sized for the whole target, so its quotient's bound `ε/(1 − ε)` misses -//! it. Only equal splits are made. -use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; - -/// The target of one of `approximate_inputs` approximate inputs that -/// compose into a value with `target`, split equally in `metric`'s -/// composition (see the module doc). `None` for an exact target or an ε -/// that is not positive and finite: no approximate input meets it. One -/// input (or none) gets `target` itself. -pub fn per_input_target( - target: &AccuracyTarget, - metric: AccuracyMetric, - approximate_inputs: usize, -) -> Option { - if target.is_exact() || !(target.max_error.is_finite() && target.max_error > 0.0) { - return None; - } - if approximate_inputs <= 1 { - return Some(target.clone()); - } - let fraction = 1.0 / approximate_inputs.max(1) as f64; - let max_error = match metric { - AccuracyMetric::Relative => 1.0 - (1.0 + target.max_error).powf(-fraction), - _ => target.max_error * fraction, - }; - Some(AccuracyTarget { - max_error, - failure_probability: target.failure_probability.map(|delta| delta * fraction), - ..target.clone() - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Additive metrics split ε and δ by `n`; the `n` shares add up to the - /// target. - #[test] - fn an_additive_target_splits_epsilon_and_delta_equally() { - let target = AccuracyTarget::max_error(0.1).failure_probability(0.02); - let share = per_input_target(&target, AccuracyMetric::ABSOLUTE, 2).unwrap(); - assert_eq!( - share, - AccuracyTarget::max_error(0.05).failure_probability(0.01) - ); - assert_eq!( - per_input_target(&target, AccuracyMetric::Rank, 1), - Some(target.clone()) - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.1), AccuracyMetric::Rank, 2) - .unwrap() - .failure_probability, - None - ); - } - - /// A relative target splits so that a quotient meets it too: - /// `(1 − ε_i)^{−n} = 1 + ε`. For ε = 0.05 over two inputs each gets - /// 0.0241, and both a ratio `(a + b)/(1 − b)` and a product - /// `(1 + a)(1 + b) − 1` of them stay within 0.05; the product-only split - /// `(1 + ε)^{1/2} − 1 = 0.0247` would give the ratio 0.0507. - #[test] - fn a_relative_target_splits_so_a_quotient_meets_it() { - let target = AccuracyTarget::max_error(0.05); - let share = per_input_target(&target, AccuracyMetric::Relative, 2) - .unwrap() - .max_error; - assert!((share - (1.0 - 1.05f64.powf(-0.5))).abs() < 1e-12); - let ratio = (share + share) / (1.0 - share); - let product = (1.0 + share) * (1.0 + share) - 1.0; - assert!(ratio <= 0.05 && product <= 0.05, "{ratio} {product}"); - assert_eq!( - per_input_target(&target, AccuracyMetric::Relative, 1), - Some(target) - ); - } - - /// No approximate input meets an exact target. - #[test] - fn an_exact_target_has_no_share() { - assert_eq!( - per_input_target(&AccuracyTarget::exact(), AccuracyMetric::ABSOLUTE, 2), - None - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.0), AccuracyMetric::ABSOLUTE, 2), - None - ); - } -} diff --git a/crates/accuracy/src/composition.rs b/crates/accuracy/src/composition.rs deleted file mode 100644 index 81b647234..000000000 --- a/crates/accuracy/src/composition.rs +++ /dev/null @@ -1,1349 +0,0 @@ -//! Registered propagation rules for computations over uncertain inputs. -//! Stage 1 composes each candidate's guarantees with them -//! (`per_computation_candidates`); a rule that needs a statistic leaves an -//! `Unknown` leaf named by one of the constants below, which Stage 3 binds. -use asap_types::accuracy::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - GuaranteeSource, ProbabilityExpr, -}; - -#[cfg(test)] -use crate::requirements::meets; -use crate::PropagationStats; - -/// Rows an exact aggregation folds (per group): bounded by the series -/// count of the sources the query reads. -pub const INPUT_ROW_COUNT: &str = "input_row_count"; -/// Rows one group of a SQL fold over an approximate relation folds: not -/// bounded by the series count of the sources it reads, so plan selection -/// leaves it unbound. -pub(crate) const GROUP_ROW_COUNT: &str = "group_row_count"; -/// Whether every input value is non-negative. -pub(crate) const VALUES_NON_NEGATIVE: &str = "values_non_negative"; -/// The absolute bound of a product of two estimates, -/// `|x|·e_y + |y|·e_x + e_x·e_y`, given enforced operand domains. -pub const PRODUCT_OPERAND_DOMAINS: &str = "product_operand_domains"; -/// The absolute bound of a quotient with an approximate denominator, -/// `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))`, given enforced operand domains -/// with the denominator's bounded away from zero. -pub(crate) const QUOTIENT_OPERAND_DOMAINS: &str = "quotient_operand_domains"; -/// `‖f‖₁` of the stream a Count-Min bound is normalized by. -pub const STREAM_L1_NORM: &str = "stream_l1_norm"; -/// `‖f‖₂` of the stream a Count Sketch bound is normalized by. -pub(crate) const STREAM_L2_NORM: &str = "stream_l2_norm"; -/// Rows of one source's stream a summary folds: with a constant update -/// weight `c`, `‖f‖₁ = |c| · rows` and `‖f‖₂ ≤ ‖f‖₁`. -pub(crate) const STREAM_ROW_COUNT: &str = "stream_row_count"; - -/// Which stream norm bounds a Count-Min or Count Sketch estimate, given its -/// summary's update weight. -#[derive(Debug, Clone, PartialEq)] -pub enum StreamNorm { - /// The weight is the source's raw value column: the source's norm - /// (`source_norm_statistic`). - Source(String), - /// A constant weight (a count): `|weight|` times the source's row count. - ConstantWeight { weight: f64, source: String }, -} - -/// The statistic for `norm` ([`STREAM_L1_NORM`] or [`STREAM_L2_NORM`]) of -/// one source's stream, e.g. `stream_l1_norm[m]`. -pub(crate) fn source_norm_statistic(norm: &str, source: &str) -> String { - format!("{norm}[{source}]") -} - -/// Whether every value folded is non-negative, for values that are raw -/// samples of `metric` or plain sums of them, e.g. -/// `values_non_negative[m]`: an indicator, 1 when proven. Plan selection -/// binds it to 1 when the data workload declares `metric` a counter or -/// non-negative, and leaves it unbound otherwise. -pub(crate) fn raw_data_samples_sign_statistic(metric: &str) -> String { - format!("{VALUES_NON_NEGATIVE}[{metric}]") -} - -/// The metric a `raw_data_samples_sign_statistic` names. -pub fn parse_raw_data_samples_sign_statistic(statistic: &str) -> Option<&str> { - statistic - .strip_prefix(VALUES_NON_NEGATIVE)? - .strip_prefix('[')? - .strip_suffix(']') -} - -/// The norm and source a `source_norm_statistic` names. -pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { - [STREAM_L1_NORM, STREAM_L2_NORM] - .into_iter() - .find_map(|norm| { - let source = statistic - .strip_prefix(norm)? - .strip_prefix('[')? - .strip_suffix(']')?; - Some((norm, source)) - }) -} - -fn additive( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> AccuracyGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); - terms.push(local.spec.error.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - local.spec.accuracy_metric, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, rule), - ) -} - -/// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors -/// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all -/// cross-products so the expression DAG is exact for any input count. -fn multiplicative( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, -) -> AccuracyGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.spec.error) - .chain(std::iter::once(&local.spec.error)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - ) -} - -fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, -) -> AccuracyGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.spec.error.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.spec.error.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.spec.failure_probability.clone(), - local.spec.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.spec.failure_probability.clone()), - }; - let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - bound, - delta, - composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - ) -} - -/// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The -/// planner composes one *per-value* child guarantee over an unknown -/// number of input rows, so both the bound and the union bound scale by -/// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is -/// not supplied. Each input's normalized bound is first converted to -/// absolute units via the statistic its metric is normalized by (also -/// unknown unless supplied); a `Rank` input has no such conversion. -fn exact_sum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_sum_relative"); - } - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.spec.accuracy_metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.spec.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), - }, - provenance, - )) -} - -/// Exact arithmetic mean over values with absolute-error guarantees. -/// Averaging cannot amplify the largest absolute input error. The event -/// that every row respects its bound is still protected conservatively -/// by a union bound over the input row count. -fn exact_average( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_average_relative"); - } - if inputs - .iter() - .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact average requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact `max`/`min` over approximate inputs of one shared metric: the -/// returned value's error is at most the largest input bound (order -/// statistics are monotone under a uniform perturbation), with -/// probability by the union bound over every input row. This bounds the -/// returned *value*; it does not identify the true winning key. -fn exact_extremum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, false, "exact_extremum_relative"); - } - let metric = inputs[0].spec.accuracy_metric; - if inputs - .iter() - .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact max/min requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(AccuracyGuarantee::new( - metric, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// An exact sum, mean, max or min of values each within relative `α_i` -/// is within relative `max α_i`. A max or min holds whatever the signs; -/// a sum or mean only over non-negative values (no cancellation), so -/// with `needs_sign` and the sign unknown the bound is the symbolic -/// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound -/// over every input row. -fn relative_fold( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - needs_sign: bool, - rule: &str, -) -> Result { - if needs_sign && stats.values_non_negative == Some(false) { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "relative error of a sum over signed values is unbounded".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let bound = if needs_sign && stats.values_non_negative.is_none() { - match &stats.sign_statistic { - // The sign of one metric's samples: the bound, gated on it. - Some(statistic) => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - BoundExpr::Product { - factors: vec![ - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - BoundExpr::Unknown { - statistic: statistic.clone(), - }, - ], - } - } - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - } - } - } - } else { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - }; - provenance.extend(composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact aggregation"), - rule, - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - bound, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact division of two relative-value estimates (an exact operand has -/// bound 0). If the numerator is within relative `a` and the denominator -/// within `b < 1`, the ratio is within -/// `max((1 + a)/(1 − b) − 1, 1 − (1 − a)/(1 + b)) = (a + b)/(1 − b)`, and -/// fails with the union bound of their failure probabilities. No operand -/// domain is needed: a relative bound below 1 keeps a zero denominator at -/// zero and a numerator's sign, so the true and estimated ratios are then -/// the same non-finite value. Both bounds must be constants: a symbolic one -/// has no rule. -/// Float64 overflow and underflow of the quotient are not bounded, as for a -/// product. -fn exact_division( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], -) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs.iter().any(|input| { - input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() - }) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].spec.error.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].spec.error.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() && numerator >= 0.0 && (0.0..1.0).contains(&denominator)) { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one".into(), - )); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact division"), - "relative_division_union_bound", - ), - )) -} - -/// Whether `guarantee` is a sketch's distinct-count estimate. -fn estimates_distinct_count(guarantee: &AccuracyGuarantee) -> bool { - let cardinality = format!( - "{:?}", - asap_types::summary_type::SketchStatistic::Cardinality - ); - guarantee.provenance.iter().any(|source| { - matches!(source, GuaranteeSource::SketchEvaluation { query, .. } if *query == cardinality) - }) -} - -/// Every input carries a relative-value guarantee. -fn all_relative(inputs: &[AccuracyGuarantee]) -> bool { - !inputs.is_empty() - && inputs - .iter() - .all(|g| g.spec.accuracy_metric == AccuracyMetric::Relative) -} - -/// `stats.input_row_count` as a bound factor, or an `Unknown` leaf named by -/// `stats.row_count_statistic` (recorded in `provenance`) when absent. -fn row_count(stats: &PropagationStats, provenance: &mut Vec) -> BoundExpr { - match stats.input_row_count { - Some(n) => BoundExpr::Constant { value: n as f64 }, - None => { - let statistic = stats.row_count_statistic.unwrap_or(INPUT_ROW_COUNT); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }); - BoundExpr::Unknown { - statistic: statistic.into(), - } - } - } -} - -/// `input` with a Count-Min ([`AccuracyMetric::ABSOLUTE_STREAM_L1`]) or Count Sketch -/// ([`AccuracyMetric::ABSOLUTE_STREAM_L2`]) bound restated as an additive absolute -/// error: the normalized bound times its stream norm, a symbolic statistic -/// until bound. The norm is the one `norm` names ([`StreamNorm`]), which -/// plan selection can bind; without one (a weight computed from the -/// values) it is a statistic nothing binds. Any other guarantee is -/// returned unchanged. -pub(crate) fn additive_as_absolute( - input: AccuracyGuarantee, - norm: Option<&StreamNorm>, -) -> AccuracyGuarantee { - if !matches!( - input.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE_STREAM_L1 | AccuracyMetric::ABSOLUTE_STREAM_L2 - ) || input.is_exact() - { - return input; - } - let mut bound = absolute_bound(&input).expect("additive metrics convert to absolute errors"); - if let (Some(norm), BoundExpr::Product { factors }) = (norm, &mut bound) { - for factor in factors { - if let BoundExpr::Unknown { statistic } = factor { - *factor = match norm { - StreamNorm::Source(source) => BoundExpr::Unknown { - statistic: source_norm_statistic(statistic, source), - }, - StreamNorm::ConstantWeight { weight, source } => BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { - value: weight.abs(), - }, - BoundExpr::Unknown { - statistic: source_norm_statistic(STREAM_ROW_COUNT, source), - }, - ], - }, - }; - } - } - } - let mut provenance = input.provenance; - unknown_statistics(&bound, &mut |statistic| { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }) - }); - let mut absolute = AccuracyGuarantee { - provenance, - ..input - }; - absolute.spec.accuracy_metric = AccuracyMetric::ABSOLUTE; - absolute.spec.error = bound; - absolute -} - -/// Every `Unknown` statistic in `bound`. -fn unknown_statistics(bound: &BoundExpr, found: &mut dyn FnMut(&str)) { - match bound { - BoundExpr::Unknown { statistic } => found(statistic), - BoundExpr::Sum { terms } - | BoundExpr::Product { factors: terms } - | BoundExpr::Max { terms } => { - for term in terms { - unknown_statistics(term, found); - } - } - BoundExpr::Scaled { inner, .. } => unknown_statistics(inner, found), - _ => {} - } -} - -/// `input`'s bound converted to absolute value units, multiplying a -/// normalized metric by the (unknown) statistic it is normalized by. `None` -/// for a metric with no such conversion (`Rank`). -fn absolute_bound(input: &AccuracyGuarantee) -> Option { - let normalizer = match input.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => return Some(input.spec.error.clone()), - AccuracyMetric::Relative => "true_value_magnitude", - AccuracyMetric::ABSOLUTE_STREAM_L1 => STREAM_L1_NORM, - AccuracyMetric::ABSOLUTE_STREAM_L2 => STREAM_L2_NORM, - // `Rank` has no distribution-free conversion to a value error; a - // metric this crate does not know has no registered conversion. - AccuracyMetric::Rank | _ => return None, - }; - if input.spec.error.is_zero() { - return Some(BoundExpr::Zero); - } - Some(BoundExpr::Product { - factors: vec![ - input.spec.error.clone(), - BoundExpr::Unknown { - statistic: normalizer.into(), - }, - ], - }) -} - -fn composed_provenance( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> Vec { - let mut provenance: Vec = inputs - .iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - }) - .collect(); - provenance.extend(local.provenance.iter().cloned()); - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: rule.into(), - }); - provenance -} - -/// The built-in composition rule: `inputs`' guarantees (in the parent's -/// child order) composed with the parent's own `local` guarantee under `op`, -/// given the statistics in `stats` (all unknown in Stage 1; plan selection -/// binds some). `Err` is the fail-closed answer: no registered rule, or a -/// missing input guarantee. It means "no rule", not "infeasible": Pass 1 -/// keeps the candidate with `guarantee = None`. -pub(crate) fn propagate( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, -) -> Result { - // Exact input: only the local guarantee remains (or the value is exact). - if inputs.iter().all(AccuracyGuarantee::is_exact) - && !matches!(op, CompositionOperator::TopKSelection) - { - return Ok(match local { - Some(local) => { - let mut out = local.clone(); - out.provenance - .extend(inputs.iter().enumerate().map(|(input_index, g)| { - GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - } - })); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - None => { - let mut out = AccuracyGuarantee::exact(format!("{op:?} over exact inputs")); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - }); - } - - let input_metrics: Vec = - inputs.iter().map(|g| g.spec.accuracy_metric).collect(); - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: input_metrics.clone(), - local_metric: local.map(|g| g.spec.accuracy_metric), - reason, - }; - // An exact input is compatible with every metric; only approximate - // inputs constrain the rule. - let approximate: Vec<&AccuracyGuarantee> = inputs.iter().filter(|g| !g.is_exact()).collect(); - let same_metric = - |metric: AccuracyMetric| approximate.iter().all(|g| g.spec.accuracy_metric == metric); - - match op { - CompositionOperator::CheckedRelativeDivision => { - if inputs.len() != 2 || local.is_some() || !same_metric(AccuracyMetric::Relative) { - return Err(unsupported( - "checked division requires two exact/relative-value operands".into(), - )); - } - let a = inputs[0] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown numerator bound".into()))?; - let b = inputs[1] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown denominator bound".into()))?; - if !(0.0..1.0).contains(&b) || a < 0.0 || !a.is_finite() { - return Err(unsupported("invalid relative division bounds".into())); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (a + b) / (1.0 - b) + 4.0 * f64::EPSILON, - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("checked floating-point division"), - "checked_relative_division_union_bound", - ), - )) - } - CompositionOperator::ApproximateAggregate => { - let local = local.ok_or_else(|| { - unsupported("approximate operator has no local guarantee to compose".into()) - })?; - if !same_metric(local.spec.accuracy_metric) { - return Err(unsupported(format!( - "no registered cross-metric rule from {input_metrics:?} to {:?}", - local.spec.accuracy_metric - ))); - } - // A distinct count over approximate values is not within the - // inputs' relative error: no rule, as for any summary whose - // estimate is not a value of its inputs. - if estimates_distinct_count(local) { - return Err(unsupported( - "no registered rule for a distinct count over approximate values".into(), - )); - } - match local.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => Ok(additive(op, inputs, local, "additive_union_bound")), - AccuracyMetric::Relative => { - if stats.values_non_negative == Some(false) { - return Err(unsupported( - "relative-error composition cannot use a known signed input".into(), - )); - } - if stats.values_non_negative.is_none() { - let mut provenance = composed_provenance( - op, - inputs, - local, - "relative_value_sign_unverified", - ); - provenance.extend(stats.evidence_provenance.clone()); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - return Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - ProbabilityExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - provenance, - )); - } - Ok(multiplicative(op, inputs, local)) - } - AccuracyMetric::Rank - | AccuracyMetric::ABSOLUTE_STREAM_L1 - | AccuracyMetric::ABSOLUTE_STREAM_L2 - | _ => Err(unsupported(format!( - "no registered same-metric composition rule for {:?} over {:?}", - local.spec.accuracy_metric, local.spec.accuracy_metric - ))), - } - } - CompositionOperator::Lipschitz { constant } => { - if !(constant.is_finite() && *constant >= 0.0) { - return Err(unsupported(format!( - "Lipschitz constant {constant} is not a finite non-negative number" - ))); - } - if inputs.len() != 1 || !same_metric(AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule is registered for exactly one absolute-error input".into(), - )); - } - if local.is_some_and(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule needs an absolute-error local guarantee".into(), - )); - } - Ok(lipschitz(op, *constant, inputs, local)) - } - CompositionOperator::ExactSum => exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => exact_extremum(op, inputs, stats), - CompositionOperator::ExactDivision => exact_division(op, inputs), - CompositionOperator::CounterRate - | CompositionOperator::InstantCounterRate - | CompositionOperator::CounterIncrease => Err(unsupported( - "counter reset detection and boundary extrapolation have no distribution-free \ - accuracy bound over approximate samples; exact samples remain exact" - .into(), - )), - CompositionOperator::TopKSelection => { - let selected = stats.topk_selected_lower_bound; - let excluded = stats.topk_excluded_upper_bound; - let delta = stats.topk_interval_failure_probability; - if selected.is_some_and(|value| !value.is_finite()) - || excluded.is_some_and(|value| !value.is_finite()) - || delta.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) - || selected - .zip(excluded) - .is_some_and(|(lower, upper)| lower <= upper) - { - return Err(unsupported( - "top-k confidence intervals overlap or contain invalid evidence".into(), - )); - } - let mut provenance = inputs - .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .collect::>(); - provenance.extend(stats.evidence_provenance.clone()); - if let Some(local) = local { - provenance.extend(local.provenance.clone()); - } - for (name, missing) in [ - ("topk_selected_lower_bound", selected.is_none()), - ("topk_excluded_upper_bound", excluded.is_none()), - ("topk_interval_failure_probability", delta.is_none()), - ] { - if missing { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: name.into(), - }); - } - } - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "topk_membership_margin_certificate".into(), - }); - let certified = selected.is_some() && excluded.is_some() && delta.is_some(); - // A certified margin makes the returned key set the true one, so - // each returned value keeps its score's error; without one, - // membership and hence the values are unknown. - let metric = approximate - .first() - .map_or(AccuracyMetric::ABSOLUTE, |score| score.spec.accuracy_metric); - if !same_metric(metric) { - return Err(unsupported( - "top-k selection needs scores of one metric".into(), - )); - } - let error = if certified { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - } else { - BoundExpr::Unknown { - statistic: "topk_membership_margin".into(), - } - }; - Ok(AccuracyGuarantee::new( - metric, - error, - delta.map_or_else( - || ProbabilityExpr::Unknown { - statistic: "topk_interval_failure_probability".into(), - }, - |value| ProbabilityExpr::Constant { value }, - ), - provenance, - )) - } - // An operator this crate does not know has no registered rule. - _ => Err(unsupported("no registered rule for this operator".into())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A source-norm statistic names its norm and source, whatever - /// characters the source holds; any other statistic is not one. - #[test] - fn source_norm_statistics_round_trip() { - for source in ["m", "a[b]", "]x[", "t.c", ""] { - for norm in [STREAM_L1_NORM, STREAM_L2_NORM] { - let statistic = source_norm_statistic(norm, source); - assert_eq!( - parse_source_norm_statistic(&statistic), - Some((norm, source)) - ); - } - } - for other in [ - INPUT_ROW_COUNT, - STREAM_L1_NORM, - "stream_l3_norm[m]", - "stream_l1_norm[m", - ] { - assert_eq!(parse_source_norm_statistic(other), None, "{other}"); - } - } - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - fn rel(bound: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Zero, - vec![], - ) - } - fn with_metric(metric: AccuracyMetric, bound: f64) -> AccuracyGuarantee { - let mut g = abs(bound, 0.0); - g.spec.accuracy_metric = metric; - g - } - #[test] - fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { - let op = CompositionOperator::CheckedRelativeDivision; - let inputs = [rel(0.01), rel(0.01)]; - let g = propagate(&op, &inputs, None, &Default::default()).unwrap(); - assert!((g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); - let mut rank = inputs[0].clone(); - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert!(propagate(&op, &[rank.clone(), rank], None, &Default::default()).is_err()); - assert!(propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()).is_err()); - } - /// A ratio of relative bounds `a`, `b` is within `(a + b)/(1 − b)`, - /// the larger of its over- and under-estimate bounds, with the union - /// bound on failure; no operand domain is needed. A denominator bound of - /// 1 or more, or a bound that is not finite and non-negative, has no - /// rule. - #[test] - fn relative_division_bounds_the_ratio_without_operand_domains() { - let mut numerator = rel(0.02); - numerator.spec.failure_probability = ProbabilityExpr::Constant { value: 0.001 }; - let got = propagate( - &CompositionOperator::ExactDivision, - &[numerator, rel(0.03)], - None, - &PropagationStats::default(), - ) - .unwrap(); - let (a, b): (f64, f64) = (0.02, 0.03); - let over = (1.0 + a) / (1.0 - b) - 1.0; - let under = 1.0 - (1.0 - a) / (1.0 + b); - assert!((got.spec.error.evaluate().unwrap() - over.max(under)).abs() < 1e-14); - assert!((got.spec.failure_probability.evaluate().unwrap() - 0.001).abs() < 1e-15); - for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &PropagationStats::default() - ) - .is_err()); - } - let rank = with_metric(AccuracyMetric::Rank, 0.01); - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rank.clone(), rank], - None, - &PropagationStats::default() - ) - .is_err()); - } - #[test] - fn exact_child_contributes_zero_error() { - let local = abs(0.05, 0.01); - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[AccuracyGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), Some(0.05)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.01)); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - } - #[test] - fn additive_bounds_and_delta_union_bound_compose() { - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.05).abs() < 1e-12); - // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::CompositionStep { rule, .. } if rule == "additive_union_bound" - ))); - } - #[test] - fn relative_error_includes_the_cross_term() { - let stats = PropagationStats { - values_non_negative: Some(true), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); - // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. - assert!((out.spec.error.evaluate().unwrap() - 0.32).abs() < 1e-12); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - } - #[test] - fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - let signed = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats { - values_non_negative: Some(false), - ..Default::default() - }, - ); - assert!(signed.is_err()); - } - #[test] - fn incompatible_metrics_are_rejected_not_treated_as_exact() { - // HLL cardinality error under a CMS frequency guarantee. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Relative, 0.01)], - Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!( - err, - AccuracyError::UnsupportedComposition { - input_metrics, - local_metric: Some(AccuracyMetric::ABSOLUTE_STREAM_L1), - .. - } if input_metrics == vec![AccuracyMetric::Relative] - )); - // Quantile rank error under value-additive logic. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&with_metric(AccuracyMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn lipschitz_scales_the_input_bound() { - let out = propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.35).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - } - #[test] - fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!( - out.spec.error.evaluate(), - None, - "unknown row count stays unknown" - ); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" - ))); - assert!(!meets(&out, 1.0, None)); - - let known = PropagationStats { - input_row_count: Some(4), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.4).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - #[test] - fn exact_extremum_takes_the_max_bound() { - let known = PropagationStats { - input_row_count: Some(2), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.3).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - /// `max`/`min` of values each within relative `α_i` is within relative - /// `max α_i` of the true extremum, whatever the signs. - #[test] - fn exact_extremum_keeps_a_relative_value_bound() { - let out = propagate( - &CompositionOperator::ExactExtremum, - &[rel(0.01), rel(0.02)], - None, - &PropagationStats { - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(out.spec.error.evaluate(), Some(0.02)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.0)); - } - /// A sum or mean of relative-error values keeps the relative bound only - /// over values known non-negative (cancellation otherwise); with the sign - /// unknown, the bound stays symbolic, and a known sign change rejects. - #[test] - fn exact_sum_and_average_keep_a_relative_bound_only_without_a_sign_change() { - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let with = |values_non_negative| { - propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - values_non_negative, - input_row_count: Some(3), - ..Default::default() - }, - ) - }; - let known = with(Some(true)).unwrap(); - assert_eq!(known.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(known.spec.error.evaluate(), Some(0.01), "{op:?}"); - let unknown = with(None).unwrap(); - assert_eq!(unknown.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - unknown.spec.error, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into() - } - ); - assert!(with(Some(false)).is_err(), "{op:?}"); - } - } - /// Over the samples of one metric, an unknown sign is named by that - /// metric ([`raw_data_samples_sign_statistic`]) and gates the bound: binding it - /// to 1 (proven) gives the relative bound, and left unbound it stays - /// unknown. - #[test] - fn a_sign_over_one_metrics_samples_gates_the_relative_bound() { - let statistic = raw_data_samples_sign_statistic("m"); - assert_eq!(parse_raw_data_samples_sign_statistic(&statistic), Some("m")); - assert_eq!( - parse_raw_data_samples_sign_statistic(VALUES_NON_NEGATIVE), - None - ); - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let out = propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - sign_statistic: Some(statistic.clone()), - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), None, "{op:?}"); - let proven = out.bind(&|s| (s == statistic).then_some(1.0)); - assert_eq!(proven.spec.error.evaluate(), Some(0.01), "{op:?}"); - let other = out.bind(&|s| (s == raw_data_samples_sign_statistic("n")).then_some(1.0)); - assert_eq!(other.spec.error.evaluate(), None, "{op:?}"); - } - } - - #[test] - fn exact_average_has_its_own_absolute_error_rule() { - let out = propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(out.spec.error.evaluate(), Some(0.25)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.04)); - } - #[test] - fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - - let certified = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(certified.spec.error.evaluate(), Some(0.1)); - assert_eq!(certified.spec.failure_probability.evaluate(), Some(0.005)); - - let overlapping = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(100.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ); - assert!(overlapping.is_err()); - - let partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert!(partial.has_unknown()); - assert_eq!(partial.spec.failure_probability.evaluate(), Some(0.005)); - - let invalid_partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(f64::NAN), - ..Default::default() - }, - ); - assert!(invalid_partial.is_err()); - } -} diff --git a/crates/accuracy/src/error_propagation.rs b/crates/accuracy/src/error_propagation.rs index d3d755f4e..c369f6d79 100644 --- a/crates/accuracy/src/error_propagation.rs +++ b/crates/accuracy/src/error_propagation.rs @@ -2,26 +2,303 @@ //! consumer's output error from its inputs' errors (concepts doc, section 4). use asap_types::accuracy::AccuracyMetric; -use asap_types::ir::Operator; -use asap_types::stages::MemoGroupId; +use asap_types::ir::operator::arguments::BinaryOpKind; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ScalarValue}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, BinaryOperator, NonASAPOp, Operator, ScalarExpr}; +use asap_types::stages::{ColumnLineage, Memo, MemoGroupId}; +use asap_types::summary_type::ExactKind; /// $g_c$'s linear bound for errors in `metric`, each input's and the /// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the -/// operator and its arguments. `None` when the consumer has no error -/// contract for `metric`: an input whose error is in `metric` then gives -/// no edge (the metric rule). No factor is guessed. +/// operator, its arguments and, for a `Filter`, its input group's column +/// lineage in `memo`. `None` when the consumer has no error contract for +/// `metric`: an edge from an input whose error is in `metric` then fails +/// (the metric rule). No factor is guessed; every contract keeps the metric +/// (concepts doc, section 4, "Local propagation"): +/// - value-preserving (`Sort`, `TimeRange`, `TimeShift`, `Concat`, a +/// readout, a finalize, a `SummaryMerge`): $\beta = 1$ in any metric; +/// - a `Project` whose items reading an estimate ([`reads_an_estimate`]) +/// each keep its error ([`keeps_error`]): $\beta = 1$, in any metric for +/// a copied column, in absolute error for `abs`, `clamp`, `clamp_min`, +/// `clamp_max` (also as the PromQL frontend's `promql_` functions) and a +/// negation, in relative error for a product with or quotient by a +/// nonzero literal (`q * 2`, `q / 1000`); +/// - a `Filter` whose predicate reads only columns copied from a source +/// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric; +/// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$), +/// min and max (absolute or relative, $\beta = 1$) and avg (absolute, +/// $\beta = 1$), without measure filters or `HAVING`, whose group-by +/// keys are copied from a source as a `Filter`'s predicate columns are +/// (a count by an estimate has no bound); +/// - `+` and `-`: absolute, $\beta = 1$; `/`: relative, +/// $\beta = 1/(1-b)$ on both, the denominator capped at $b$; `*`: +/// relative, $(1+b, 1)$, the right operand capped at $b$ +/// ($(1+x)(1+y)-1 \le (1+b)\,x + y$ for $y \le b$), with $b$ = +/// [`RELATIVE_OPERAND_VALIDITY_CAP`]. +/// +/// Every other operator (a filter comparing a computed value, a ranking, +/// a `sum`, a comparison, another function) has none. pub fn error_propagation( + memo: &Memo, consumer: &Operator, metric: AccuracyMetric, ) -> Option { - let _ = (consumer, metric); - todo!("#1207 PR 4: the error contract of each operator, per metric") + let inputs = consumer.children().len(); + let uniform = |factor: f64| ErrorPropagation { + factor_per_input: vec![factor; inputs], + validity_cap_per_input: vec![None; inputs], + }; + let absolute = matches!(metric, AccuracyMetric::Absolute { .. }); + let extremum_metric = absolute || metric == AccuracyMetric::Relative; + match consumer { + Operator::NonASAP(op) => match op { + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::Concat { .. } => Some(uniform(1.0)), + NonASAPOp::Project { cols, child } => cols + .iter() + .filter(|item| reads_an_estimate(memo, *child, &item.expr)) + .all(|item| keeps_error(&item.expr, metric)) + .then(|| uniform(1.0)), + NonASAPOp::Filter { pred, child } => (pred.0.operator_refs().is_empty() + && copied_from_a_source(memo, *child, pred.0.columns_referenced())) + .then(|| uniform(1.0)), + NonASAPOp::Aggregate { + measures, + filters, + having: None, + reduction, + child, + .. + } if filters.iter().all(Option::is_none) + && !measures.is_empty() + && keys_copied_from_a_source(memo, *child, reduction) => + { + let mut factor: f64 = 0.0; + for measure in measures { + match measure { + AggIntent::Count => {} + AggIntent::Min { .. } | AggIntent::Max { .. } if extremum_metric => { + factor = factor.max(1.0) + } + AggIntent::Avg { .. } if absolute => factor = factor.max(1.0), + _ => return None, + } + } + Some(uniform(factor)) + } + NonASAPOp::BinaryOp { + operator: + BinaryOperator { + kind: BinaryOpKind::Arithmetic(kind), + .. + }, + return_bool: false, + .. + } => { + let b = RELATIVE_OPERAND_VALIDITY_CAP; + match (kind, metric) { + (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, _) if absolute => { + Some(uniform(1.0)) + } + (ArithmeticOpKind::Div, AccuracyMetric::Relative) => Some(ErrorPropagation { + factor_per_input: vec![1.0 / (1.0 - b); 2], + validity_cap_per_input: vec![None, Some(b)], + }), + (ArithmeticOpKind::Mul, AccuracyMetric::Relative) => Some(ErrorPropagation { + factor_per_input: vec![1.0 + b, 1.0], + validity_cap_per_input: vec![None, Some(b)], + }), + _ => None, + } + } + _ => None, + }, + Operator::ASAP(op) => match op { + ASAPOp::SummaryEstimate { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::SummaryMerge { .. } => Some(uniform(1.0)), + ASAPOp::SummaryAgg { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + reduction, + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + reduction, + child, + .. + } if keys_copied_from_a_source(memo, *child, reduction) => match kind { + ExactKind::Count => Some(uniform(0.0)), + ExactKind::Min | ExactKind::Max if extremum_metric => Some(uniform(1.0)), + _ => None, + }, + _ => None, + }, + } } +/// The metrics among `metrics` that `consumer`'s error contract +/// ([`error_propagation`]) carries to its inputs: those an input of it may +/// be off in when its output accepts `metrics`. +pub fn metrics_carried( + memo: &Memo, + consumer: &Operator, + metrics: &[AccuracyMetric], +) -> Vec { + metrics + .iter() + .copied() + .filter(|&metric| error_propagation(memo, consumer, metric).is_some()) + .collect() +} + +/// Whether each of `columns` of `group`'s output is copied from a source +/// (lineage `SourceColumn`: a label or key), not computed: an estimate's +/// error can change which rows a predicate or a grouping on it selects. +fn copied_from_a_source( + memo: &Memo, + group: MemoGroupId, + columns: impl IntoIterator, +) -> bool { + let lineage = &memo.groups()[group.0 as usize] + .group_properties + .column_lineage; + columns + .into_iter() + .all(|column| matches!(lineage.get(column), Some(ColumnLineage::SourceColumn(_)))) +} + +/// Whether every key `reduction` groups `child`'s rows by is copied from a +/// source ([`copied_from_a_source`]): `by` keys as listed; a `without` +/// grouping keys by every column of `child` but the time, the sample value +/// and the excluded ones; a per-entity reduction keeps each row's group. +fn keys_copied_from_a_source(memo: &Memo, child: MemoGroupId, reduction: &Reduction) -> bool { + let Reduction::Reduce(by) = reduction else { + return true; + }; + if !by.is_without() { + return copied_from_a_source(memo, child, by.keys().iter().copied()); + } + let schema = &memo.groups()[child.0 as usize].group_properties.schema; + let value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); + copied_from_a_source( + memo, + child, + (0..schema.fields.len()).filter(|&column| { + schema.time_index != Some(column) + && value != Some(column) + && !by.keys().contains(&column) + }), + ) +} + +/// Whether `expr`, an item of a `Project` over `child`, reads a value that +/// may carry an estimate's error: a column of `child` that is neither +/// copied from a source (lineage `SourceColumn`: a label, a key, the series +/// identity) nor its time, or another operator's result. A PromQL function +/// of the value replaces only the value; the time and labels it copies +/// (and `promql_drop_metric_name` of the series identity) read no estimate. +fn reads_an_estimate(memo: &Memo, child: MemoGroupId, expr: &ScalarExpr) -> bool { + let properties = &memo.groups()[child.0 as usize].group_properties; + !expr.operator_refs().is_empty() + || expr.columns_referenced().into_iter().any(|column| { + properties.schema.time_index != Some(column) + && !matches!( + properties.column_lineage.get(column), + Some(ColumnLineage::SourceColumn(_)) + ) + }) +} + +/// Whether a `Project` item `expr` keeps the error in `metric` of the +/// column it reads, with $\beta = 1$: a copy of a column or a literal, or +/// the series identity without its metric name (`promql_drop_metric_name`, +/// which changes a label, not a value), in any metric; a 1-Lipschitz +/// function of a column ([`one_lipschitz`]) in absolute error; a column +/// times or divided by a nonzero literal in relative error. +fn keeps_error(expr: &ScalarExpr, metric: AccuracyMetric) -> bool { + match expr { + ScalarExpr::Column(_) | ScalarExpr::Literal(_) => true, + ScalarExpr::FunctionCall { name, args } if name == "promql_drop_metric_name" => { + matches!(args[..], [ScalarExpr::Column(_)]) + } + _ if matches!(metric, AccuracyMetric::Absolute { .. }) => one_lipschitz(expr), + _ if metric == AccuracyMetric::Relative => scaled_by_a_constant(expr), + _ => false, + } +} + +/// Whether `expr` changes a column's value by at most its error: `abs`, +/// `clamp`, `clamp_min` or `clamp_max` (or the PromQL frontend's +/// `promql_abs`, …) of a column with literal bounds, or a column's +/// negation (1-Lipschitz). +fn one_lipschitz(expr: &ScalarExpr) -> bool { + match expr { + ScalarExpr::Negative { expr, .. } => matches!(**expr, ScalarExpr::Column(_)), + ScalarExpr::FunctionCall { name, args } => { + let name = name.to_ascii_lowercase(); + matches!( + name.strip_prefix("promql_").unwrap_or(&name), + "abs" | "clamp" | "clamp_min" | "clamp_max" + ) && matches!(args.first(), Some(ScalarExpr::Column(_))) + && args[1..] + .iter() + .all(|arg| matches!(arg, ScalarExpr::Literal(_))) + } + _ => false, + } +} + +/// Whether `expr` is a column times a nonzero literal (either side) or +/// divided by one: $c\hat{x}$ is off from $cx$ by the same share as +/// $\hat{x}$ from $x$ (a unit conversion, `q * 1000`). +fn scaled_by_a_constant(expr: &ScalarExpr) -> bool { + let nonzero = |expr: &ScalarExpr| match expr { + ScalarExpr::Literal(ScalarValue::Float64(x)) => x.is_finite() && *x != 0.0, + ScalarExpr::Literal(ScalarValue::Int64(x)) => *x != 0, + _ => false, + }; + let column = |expr: &ScalarExpr| matches!(expr, ScalarExpr::Column(_)); + match expr { + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + .. + } => (column(left) && nonzero(right)) || (nonzero(left) && column(right)), + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left, + right, + .. + } => column(left) && nonzero(right), + _ => false, + } +} + +/// $b$ of a ratio's denominator and a product's right operand: the +/// relative error under which [`error_propagation`]'s linear bound holds, +/// $\varepsilon_{X/Y} \le (\varepsilon_X + \varepsilon_Y) / (1 - b)$ +/// (concepts doc, section 4). 0.1 is the weakest error of every +/// algorithm's list, so it never excludes a listed θ. +const RELATIVE_OPERAND_VALIDITY_CAP: f64 = 0.1; + /// The local error propagation of one consumer: its output error is at /// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays /// within its validity cap (concepts doc, section 4; approved 2026-11-05). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] pub struct ErrorPropagation { /// $\beta_{c,i}$ per input, for that input's metric. pub factor_per_input: Vec, @@ -30,3 +307,296 @@ pub struct ErrorPropagation { /// denominator). pub validity_cap_per_input: Vec>, } + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::operator::arguments::Reduction; + use asap_types::ir::scalar::CompareOpKind; + use asap_types::ir::schema::{DataType, PROMQL_SERIES_IDENTITY}; + use asap_types::ir::{ExprSemantics, Predicate, ProjectItem}; + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + use asap_types::test_support::memo_fixtures::{aggregate, arithmetic, compare, readout, scan}; + use asap_types::workload::DataWorkload; + + /// Constant error contracts. A `Filter` whose + /// predicate reads only columns copied from a source keeps or drops + /// whole rows: β = 1 in every metric. Over a readout of a KLL state of + /// `latency` by `job`, `job = "api"` reads such a column; `value > 0.5` + /// compares the computed estimate and has no contract. A `Project` of + /// `abs`, `clamp` or a negation of the value changes it by at most its + /// error: β = 1 for absolute errors only; one scaling it by a nonzero + /// constant keeps its relative error: β = 1 for relative errors. Only + /// the items reading the estimate are judged, so the series identity + /// `promql_drop_metric_name` computes does not count. + #[test] + fn key_filters_and_value_preserving_projects_have_unit_contracts() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.9 }, state)) + .unwrap(); + let lineage = &memo.groups()[readout.0 as usize] + .group_properties + .column_lineage; + let column_where = |copied: bool| { + lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied) + .expect("the readout has a label and an estimate") + }; + let (job, estimate) = (column_where(true), column_where(false)); + let filter = |pred| { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child: readout, + }) + }; + let by_job = filter(compare( + job, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )); + let by_value = filter(compare( + estimate, + CompareOpKind::Gt, + ScalarValue::Float64(0.5), + ExprSemantics::Promql, + )); + for metric in AccuracyMetric::ALL { + assert_eq!( + error_propagation(&memo, &by_job, metric) + .map(|propagation| propagation.factor_per_input), + Some(vec![1.0]), + "key-only filter, {metric:?}" + ); + assert_eq!( + error_propagation(&memo, &by_value, metric), + None, + "a filter comparing the estimate, {metric:?}" + ); + } + // A `Project` over the readout, as the PromQL frontend lowers a + // function of the value: the time and every label copied, the series + // identity through `promql_drop_metric_name`, the estimate replaced. + let value = || Box::new(ScalarExpr::Column(estimate)); + let literal = |x: f64| ScalarExpr::Literal(ScalarValue::Float64(x)); + let project_value = |expr: ScalarExpr| { + let fields = &memo.groups()[readout.0 as usize] + .group_properties + .schema + .fields; + Operator::NonASAP(NonASAPOp::Project { + cols: fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: if column == estimate { + expr.clone() + } else if field.name == PROMQL_SERIES_IDENTITY { + ScalarExpr::FunctionCall { + name: "promql_drop_metric_name".into(), + args: vec![ScalarExpr::Column(column)], + } + } else { + ScalarExpr::Column(column) + }, + }) + .collect(), + child: readout, + }) + }; + let factor = |project: &Operator, metric| { + error_propagation(&memo, project, metric) + .map(|propagation| propagation.factor_per_input) + }; + let one_lipschitz = [ + ScalarExpr::FunctionCall { + name: "promql_abs".into(), + args: vec![*value()], + }, + ScalarExpr::FunctionCall { + name: "promql_clamp".into(), + args: vec![*value(), literal(0.0), literal(100.0)], + }, + ScalarExpr::FunctionCall { + name: "promql_clamp_max".into(), + args: vec![*value(), literal(5.0)], + }, + ScalarExpr::Negative { + expr: value(), + semantics: ExprSemantics::Promql, + }, + ]; + for expr in one_lipschitz { + let project = project_value(expr.clone()); + assert_eq!( + factor(&project, AccuracyMetric::ABSOLUTE), + Some(vec![1.0]), + "{expr:?}" + ); + assert_eq!(factor(&project, AccuracyMetric::Relative), None, "{expr:?}"); + } + // Scaling by a nonzero constant keeps a relative error: `q / 2`, + // `q * 2`, `2 * q`; scaling by 0 has no contract. + let scaled = |op, left, right| ScalarExpr::Arithmetic { + op, + left: Box::new(left), + right: Box::new(right), + semantics: ExprSemantics::Promql, + }; + for expr in [ + scaled(ArithmeticOpKind::Div, *value(), literal(2.0)), + scaled(ArithmeticOpKind::Mul, *value(), literal(2.0)), + scaled(ArithmeticOpKind::Mul, literal(2.0), *value()), + ] { + let project = project_value(expr.clone()); + assert_eq!( + factor(&project, AccuracyMetric::Relative), + Some(vec![1.0]), + "{expr:?}" + ); + assert_eq!(factor(&project, AccuracyMetric::Rank), None, "{expr:?}"); + } + let times_zero = project_value(scaled(ArithmeticOpKind::Mul, *value(), literal(0.0))); + assert_eq!(factor(×_zero, AccuracyMetric::Relative), None); + // Copying the value (and dropping the metric name from the series + // identity) keeps it in every metric. + let copied = project_value(*value()); + for metric in AccuracyMetric::ALL { + assert_eq!(factor(&copied, metric), Some(vec![1.0]), "{metric:?}"); + } + } + + /// A count (and min, max, avg) grouped by a column that is itself an + /// estimate has no contract: an estimate's error moves rows between + /// groups, so a count by it has no bound, as for a filter comparing it. + /// Over a readout of a KLL state of `latency` by `job`, `count by + /// (estimate)` and an exact count accumulator by it have none, while + /// `count by (job)` reads no value (factor 0). + #[test] + fn a_reduction_keyed_by_an_estimate_has_no_contract() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.99 }, state)) + .unwrap(); + let lineage = &memo.groups()[readout.0 as usize] + .group_properties + .column_lineage; + let column_where = |copied: bool| { + lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied) + .expect("the readout has a label and an estimate") + }; + let (job, estimate) = (column_where(true), column_where(false)); + let exact_count_by = |key: usize| { + Operator::ASAP(ASAPOp::SummaryAgg { + child: readout, + summary_type: FieldDataType::exact(ExactKind::Count), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), + reduction: Reduction::by(vec![key]), + filter: None, + }) + }; + for metric in AccuracyMetric::ALL { + for measure in [ + AggIntent::Count, + AggIntent::Min { col: None }, + AggIntent::Max { col: None }, + AggIntent::Avg { col: None }, + ] { + assert_eq!( + error_propagation( + &memo, + &aggregate(Reduction::by(vec![estimate]), measure.clone(), readout), + metric + ), + None, + "{measure:?} by the estimate, {metric:?}" + ); + } + assert_eq!( + error_propagation(&memo, &exact_count_by(estimate), metric), + None, + "an exact count by the estimate, {metric:?}" + ); + for count_by_job in [ + aggregate(Reduction::by(vec![job]), AggIntent::Count, readout), + exact_count_by(job), + ] { + assert_eq!( + error_propagation(&memo, &count_by_job, metric) + .map(|propagation| propagation.factor_per_input), + Some(vec![0.0]), + "a count by job, {metric:?}" + ); + } + } + } + + /// A ratio carries relative error with factor 1/(1 − 0.1) on both + /// inputs and caps its denominator at 0.1; a count reads no value + /// (factor 0); a sum has no contract. + #[test] + fn error_contracts_of_a_ratio_a_count_and_a_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[("job", DataType::Utf8)], vec![])) + .unwrap(); + let divide = arithmetic(ArithmeticOpKind::Div, g1, g1); + assert_eq!( + error_propagation(&memo, ÷, AccuracyMetric::Relative), + Some(ErrorPropagation { + factor_per_input: vec![1.0 / 0.9; 2], + validity_cap_per_input: vec![None, Some(0.1)], + }) + ); + for metric in [AccuracyMetric::ABSOLUTE, AccuracyMetric::Rank] { + assert_eq!(error_propagation(&memo, ÷, metric), None); + } + let by_job = || Reduction::by(vec![2]); + assert_eq!( + error_propagation( + &memo, + &aggregate(by_job(), AggIntent::Count, g1), + AccuracyMetric::Rank + ) + .map(|propagation| propagation.factor_per_input), + Some(vec![0.0]) + ); + assert_eq!( + error_propagation( + &memo, + &aggregate(by_job(), AggIntent::Sum { col: Some(1) }, g1), + AccuracyMetric::ABSOLUTE + ), + None + ); + } +} diff --git a/crates/accuracy/src/function_rules.rs b/crates/accuracy/src/function_rules.rs deleted file mode 100644 index 1a91edea8..000000000 --- a/crates/accuracy/src/function_rules.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! The accuracy composition rule of each exact function applied to an -//! approximate input, shared by Pass 1's guarantee composition and the -//! accuracy algebra. -use asap_types::accuracy::CompositionOperator; -use asap_types::ir::operator::AggIntent; - -/// The composition operator that propagates an approximate input's guarantee -/// through `intent`; `None` for a function with no registered rule over -/// approximate input, whose callers must fail closed. -pub(crate) fn approximate_input_composition(intent: &AggIntent) -> Option { - Some(match intent { - AggIntent::Sum { .. } => CompositionOperator::ExactSum, - AggIntent::Min { .. } | AggIntent::Max { .. } => CompositionOperator::ExactExtremum, - AggIntent::Avg { .. } => CompositionOperator::ExactAverage, - AggIntent::Rate => CompositionOperator::CounterRate, - AggIntent::IRate => CompositionOperator::InstantCounterRate, - AggIntent::Increase => CompositionOperator::CounterIncrease, - _ => return None, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn counter_functions_have_distinct_definition_rules() { - assert_eq!( - approximate_input_composition(&AggIntent::Rate), - Some(CompositionOperator::CounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::IRate), - Some(CompositionOperator::InstantCounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::Increase), - Some(CompositionOperator::CounterIncrease) - ); - } -} diff --git a/crates/accuracy/src/guarantee/binomial.rs b/crates/accuracy/src/guarantee/binomial.rs new file mode 100644 index 000000000..d2d0b85b6 --- /dev/null +++ b/crates/accuracy/src/guarantee/binomial.rs @@ -0,0 +1,26 @@ +//! The binomial tail the per-row failure bounds share: a structure whose +//! independent rows each miss with probability `p` misses when at least +//! `k` of its `n` rows do. + +/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded +/// up by a relative 1e-12 so rounding never certifies a smaller tail. +pub(super) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { + if k == 0 || p >= 1.0 { + return 1.0; + } + if k > n || p <= 0.0 { + return 0.0; + } + let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); + let mut ln_choose = 0.0; + let mut tail = 0.0; + for i in 0..=n { + if i > 0 { + ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); + } + if i >= k { + tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); + } + } + (tail * (1.0 + 1e-12)).min(1.0) +} diff --git a/crates/accuracy/src/guarantee/cardinality.rs b/crates/accuracy/src/guarantee/cardinality.rs index 620d93742..28361b3b3 100644 --- a/crates/accuracy/src/guarantee/cardinality.rs +++ b/crates/accuracy/src/guarantee/cardinality.rs @@ -1,6 +1,6 @@ //! KMV's provable distinct-count bound. //! -//! # KMV (#694) +//! # KMV //! //! `asap_sketchlib`'s KMV keeps the `k` smallest distinct 64-bit hashes of //! its items and estimates (`src/sketches/kmv.rs`): @@ -25,38 +25,49 @@ //! //! Neither bound depends on `N`, so no distinct bound is needed: the //! failure probability ([`kmv_failure_probability`]) is their sum, per -//! output row (each group's estimate). Sizing (test-only -//! `sizing_oracle::kmv_k`) takes the smallest `k` with that sum at most -//! `δ`, about -//! `1 + (1+ε)(2+ε)·ln(2/δ)/ε²` (below `3·ln(2/δ)/ε² + 1` for `ε ≤ 1`). +//! output row (each group's estimate). The guarantee at a failure +//! probability δ ([`guarantee`]) is the smallest `ε` with that sum at +//! most δ. use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. -pub(super) const KMV_MIN_K: u32 = 2; +const KMV_MIN_K: u32 = 2; +/// The smallest relative error `k` misses with probability at most +/// `failure_probability` ([`kmv_failure_probability`], by bisection); +/// `None` when no error up to 1 does. pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, + failure_probability: f64, ) -> Option { - match params { - SketchParams::Kmv { k } => { - // Its relative standard error. - let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::Relative, - eps, - ProbabilityExpr::Constant { - value: kmv_failure_probability(*k, eps), - }, - "kmv_exact_below_k_chernoff_v1", - )) + let SketchParams::Kmv { k } = params else { + return None; + }; + if kmv_failure_probability(*k, 1.0) > failure_probability { + return None; + } + let (mut met, mut missed) = (1.0, 0.0); + for _ in 0..60 { + let eps = (met + missed) / 2.0; + if kmv_failure_probability(*k, eps) <= failure_probability { + met = eps; + } else { + missed = eps; } - _ => None, } + Some(super::bounded_guarantee( + algorithm, + params, + query, + AccuracyMetric::Relative, + met, + ProbabilityExpr::Constant { + value: failure_probability, + }, + "kmv_exact_below_k_chernoff_v1", + )) } /// The probability that `asap_sketchlib`'s KMV with `k` misses relative diff --git a/crates/accuracy/src/guarantee/cms.rs b/crates/accuracy/src/guarantee/cms.rs index 7333ea698..e7760b68a 100644 --- a/crates/accuracy/src/guarantee/cms.rs +++ b/crates/accuracy/src/guarantee/cms.rs @@ -28,24 +28,6 @@ pub(super) fn guarantee( mod tests { use super::*; - #[test] - fn local_guarantee_inverts_frequency_sizing() { - use crate::guarantee::sizing_oracle::size_params; - let c = asap_types::ir::operator::agg_intent::default_cardinality(); - let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = sketch_guarantee( - &SketchAlgorithm::Cms, - ¶ms, - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(meets(&g, 0.01, Some(0.001))); - } - #[test] fn heap_evaluation_retains_frequency_metric() { let cms_heap = SketchParams::CmsWithHeap { @@ -53,10 +35,11 @@ mod tests { depth: 5, heap_size: 10, }; - let topk_frequency = sketch_guarantee( + let topk_frequency = guarantee_of_parameters( &SketchAlgorithm::CmsWithHeap, &cms_heap, &SketchStatistic::TopK { k: 10 }, + 1.0, ) .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( diff --git a/crates/accuracy/src/guarantee/count_sketch.rs b/crates/accuracy/src/guarantee/count_sketch.rs index 9c6ec7011..a924d11f3 100644 --- a/crates/accuracy/src/guarantee/count_sketch.rs +++ b/crates/accuracy/src/guarantee/count_sketch.rs @@ -28,7 +28,7 @@ pub(super) fn guarantee( /// probability at most `q/ε² = q w/3` (Chebyshev, `Var ≤ q F2`), with `q` /// the column collision probability ([`column_collision`]): `1/3` to /// within `w/(3 · 2^32)`. -pub(super) fn row_failure_probability(width: u32) -> f64 { +fn row_failure_probability(width: u32) -> f64 { column_collision(width) * f64::from(width.max(1)) / 3.0 } @@ -45,7 +45,7 @@ fn column_collision(cols: u32) -> f64 { /// The median of an odd `depth` of independent rows misses only when at /// least `(depth + 1)/2` rows miss: `P(Binomial(depth, p) ≥ (depth+1)/2)`. -pub(super) fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { +fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { if depth == 0 || depth.is_multiple_of(2) { return None; } @@ -82,62 +82,36 @@ mod tests { ); } - /// Seeded end to end: the sized sketch (ε = 0.05, δ = 0.01, width 1200, - /// depth 47) answers each key of a skewed stream within `ε ‖f‖₂`, at - /// most a δ share of keys missing. - #[test] - fn sized_sketch_meets_its_guarantee_on_a_seeded_stream() { - use crate::guarantee::sizing_oracle::{count_sketch_depth, count_sketch_width}; - use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; - let (eps, delta) = (0.05, 0.01); - let width = count_sketch_width(eps); - let depth = count_sketch_depth(width, delta); - let mut sketch = CountSketch::, RegularPath>::with_dimensions( - depth as usize, - width as usize, - ); - // 20 000 keys, key i weighted 1e6 / (i + 1) (Zipf, s = 1), in a - // seeded order. - let keys = 20_000u64; - let weight = |key: u64| (1e6 / (key + 1) as f64).round(); - let mut state = 0x9E37_79B9_7F4A_7C15u64; - let mut order: Vec = (0..keys).collect(); - for i in (1..order.len()).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(i, (state % (i as u64 + 1)) as usize); - } - for &key in &order { - sketch.update_weighted(&DataInput::U64(key), weight(key)); - } - let l2 = (0..keys).map(|k| weight(k).powi(2)).sum::().sqrt(); - let bound = (3.0 / f64::from(width)).sqrt() * l2; - let misses = (0..keys) - .filter(|&k| (sketch.estimate(&DataInput::U64(k)) - weight(k)).abs() > bound) - .count(); - assert!(misses as f64 <= delta * keys as f64, "{misses} misses"); - } - #[test] fn count_sketch_uses_an_l2_guarantee() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let intent = default_cardinality(); - let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = sketch_guarantee( + // ε = √(3/30 000) = 0.01; 47 rows fail together with probability ≤ 0.01. + let count_sketch = SketchParams::CountSketch { + width: 30_000, + depth: 47, + }; + let guarantee = guarantee_of_parameters( &SketchAlgorithm::CountSketch, &count_sketch, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: Some("1".into()), }, + 1.0, ) .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( guarantee.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L2 ); - assert!(meets(&guarantee, 0.01, Some(0.01))); + assert!(guarantee + .spec + .error + .evaluate() + .is_some_and(|error| error <= target_limit(0.01))); + assert!(guarantee + .spec + .failure_probability + .evaluate() + .is_some_and(|failure_probability| failure_probability <= target_limit(0.01))); } } diff --git a/crates/accuracy/src/guarantee/hll.rs b/crates/accuracy/src/guarantee/hll.rs index fc483ec2b..ab24a4e22 100644 --- a/crates/accuracy/src/guarantee/hll.rs +++ b/crates/accuracy/src/guarantee/hll.rs @@ -1,8 +1,4 @@ -//! Estimator-specific confidence for classic HLL's linear-counting branch. -//! -//! This is conditional on independent uniform bucket hashes and an enforced -//! upper bound on distinct items in the complete evaluation population (including -//! all merged tumbling windows). It is not an RSE-to-normal conversion or an ERP fit. +//! HLL's relative standard error, without a failure probability. use super::*; @@ -30,345 +26,3 @@ pub(super) fn generic_guarantee( fn relative_standard_error(precision: u8) -> f64 { 1.04 / 2f64.powi(i32::from(precision)).sqrt() } - -/// Names classic HLL's failure probability at `precision` and a relative -/// error as a function of `max_distinct[source]`, the deployment's bound on -/// distinct items per evaluation of `source`, which plan selection binds. -const CLASSIC_FAILURE_PROBABILITY: &str = "classic_hll_failure_probability"; -const MAX_DISTINCT: &str = "max_distinct"; - -/// The relative error a classic HLL's guarantee states: the one its -/// precision was sized for, else its relative standard error. -pub fn classic_relative_error(precision: u8, relative_error: Option) -> f64 { - relative_error.unwrap_or_else(|| relative_standard_error(precision)) -} - -/// The statistic classic HLL's failure probability at `precision` and -/// `relative_error` over `source` is left as (`classic_guarantee`). -/// `relative_error` is written in Rust's shortest round-trip form, so it -/// parses back to the same value. -pub fn classic_failure_statistic(precision: u8, relative_error: f64, source: &str) -> String { - format!( - "{CLASSIC_FAILURE_PROBABILITY}[p={precision},eps={relative_error}]({MAX_DISTINCT}[{source}])" - ) -} - -/// The precision, relative error and source a classic HLL failure -/// statistic (`classic_failure_statistic`) names. -pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str)> { - let rest = statistic - .strip_prefix(CLASSIC_FAILURE_PROBABILITY)? - .strip_prefix("[p=")?; - let (params, rest) = rest.split_once(']')?; - let (precision, relative_error) = params.split_once(",eps=")?; - let source = rest - .strip_prefix('(')? - .strip_prefix(MAX_DISTINCT)? - .strip_prefix('[')? - .strip_suffix("])")?; - Some(( - precision.parse().ok()?, - relative_error.parse().ok()?, - source, - )) -} - -/// Classic HLL's guarantee for a distinct count over `source`'s own rows -/// (one instance per group, over the source's scan at most behind filters -/// and windows): relative error at most the one its precision was sized for -/// ([`classic_relative_error`]), with a failure probability -/// (`ClassicHllConfidence`) at that error left symbolic in the source's -/// distinct bound ([`classic_failure_statistic`]). The failure probability -/// is per output row: each group's estimate fails with at most it. `None` -/// for a precision outside the contract's. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn classic_guarantee( - precision: u8, - relative_error: Option, - source: &str, -) -> Option { - if !(4..=MAX_PRECISION).contains(&precision) { - return None; - } - let relative_error = classic_relative_error(precision, relative_error); - let statistic = classic_failure_statistic(precision, relative_error, source); - Some(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: relative_error, - }, - ProbabilityExpr::Unknown { - statistic: statistic.clone(), - }, - vec![GuaranteeSource::SketchEvaluation { - algorithm: "Hll".into(), - contract: CLASSIC_CONTRACT.into(), - params: serde_json::json!({"precision": precision, - "max_distinct": format!("{MAX_DISTINCT}[{source}]"), - "relative_error": relative_error, - "hash_assumption": "independent_uniform_buckets", - "population_scope": "complete_evaluation_including_merged_tumbling_windows"}), - query: "Cardinality".into(), - }], - )) -} - -/// The failure probability a classic HLL failure statistic -/// (`classic_failure_statistic`) names, given the -/// source's distinct bound: `1` (no certificate) when the bound is outside -/// the contract's range for this precision and relative error. -#[cfg(test)] -pub(crate) fn classic_failure_probability( - precision: u8, - relative_error: f64, - max_distinct: u64, -) -> f64 { - u32::try_from(max_distinct) - .ok() - .and_then(|n| ClassicHllConfidence::new(n, relative_error)) - .and_then(|model| model.failure_probability(precision)) - .unwrap_or(1.0) -} - -/// The largest precision classic HLL's contract covers. -pub const MAX_PRECISION: u8 = 18; - -const CLASSIC_CONTRACT: &str = "classic_hll_linear_counting_collision_bound_v1"; - -/// The largest distinct bound the contract supports at any precision: -/// `m/2` at precision 18. Evaluating a bound is O(bound). -#[cfg(test)] -pub(super) const MAX_CONTRACT_DISTINCT: u32 = 1 << 17; - -/// A finite-population contract for `m * ln(m / zero_registers)` with the -/// classic HLL small-range switch. Hashing is assumed independent and uniform. -/// The deployment must establish the population bound; observations alone do -/// not establish it. Unsupported precisions/populations return no certificate. -#[cfg(test)] -#[derive(Debug, Clone, Copy)] -pub(crate) struct ClassicHllConfidence { - max_distinct: u32, - relative_error: f64, -} - -#[cfg(test)] -impl ClassicHllConfidence { - pub(crate) fn new(max_distinct: u32, relative_error: f64) -> Option { - (max_distinct > 0 - && max_distinct <= MAX_CONTRACT_DISTINCT - && relative_error.is_finite() - && (1e-6..1.0).contains(&relative_error)) - .then_some(Self { - max_distinct, - relative_error, - }) - } - - /// Finite bound, not an asymptotic RSE fit. With N distinct hashes and K - /// occupied buckets, C=N-K collision arrivals satisfy - /// P(C>=t) <= lambda^t/t!, lambda=N(N-1)/(2m): each arrival's conditional - /// collision probability is at most (i-1)/m, and a union bound over t - /// arrivals is bounded by the t-th power of their sum divided by t!. - /// - /// N<=m/2 makes the classic raw estimate <=2*alpha_m*m<2.5m, - /// so the small-range switch always uses L=-m*ln(1-K/m). Then - /// K<=L<=N + N^2/(2(m-N)). The latter bounds overestimation - /// deterministically; underestimation implies C>epsilon*N. - /// We maximize the collision bound over EVERY integer N in the contract, - /// not just its upper endpoint (small-cardinality tails matter). - pub(super) fn failure_probability(&self, precision: u8) -> Option { - if !(4..=MAX_PRECISION).contains(&precision) { - return None; - } - let m = f64::from(1u32 << precision); - let max_n = f64::from(self.max_distinct); - // Reserve numerical slack; do not certify sub-floating-point error. - let eps = self.relative_error - 1e-8; - if max_n > m / 2.0 || max_n / (2.0 * (m - max_n)) > eps { - return None; - } - let mut log_factorial = vec![0.0; self.max_distinct as usize + 1]; - for i in 1..log_factorial.len() { - log_factorial[i] = log_factorial[i - 1] + (i as f64).ln(); - } - let mut worst = 0.0_f64; - for n in 2..=self.max_distinct { - let nf = f64::from(n); - // Including a boundary collision event is conservative. - let t = ((eps * nf).floor() as usize + 1).min(n as usize); - let lambda = nf * (nf - 1.0) / (2.0 * m); - let log_tail = (t as f64) * lambda.ln() - log_factorial[t]; - worst = worst.max(log_tail.min(0.0).exp()); - } - // Never return a spurious zero from underflow or numeric cancellation. - Some((worst * (1.0 + 1e-10) + 1e-12).min(1.0)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The failure-probability statistic names its precision, relative - /// error and source, any source name. - #[test] - fn classic_failure_statistics_round_trip() { - for source in ["flows_src_ip", "a[b]", "x])(", ""] { - for eps in [0.02, 0.1 + 0.2, 1.04 / 64.0] { - let statistic = classic_failure_statistic(12, eps, source); - assert_eq!( - parse_classic_failure_statistic(&statistic), - Some((12, eps, source)) - ); - } - } - for other in [ - "max_distinct[m]", - "classic_hll_failure_probability[p=x,eps=0.02](max_distinct[m])", - "classic_hll_failure_probability[p=12](max_distinct[m])", - ] { - assert_eq!(parse_classic_failure_statistic(other), None); - } - } - - /// The symbolic guarantee states the relative error the precision was - /// sized for and binds to the model's probability at that error; an - /// out-of-range bound certifies nothing. - #[test] - fn classic_guarantee_binds_the_distinct_bound() { - let guarantee = classic_guarantee(17, Some(0.02), "m").unwrap(); - assert_eq!(guarantee.spec.error.evaluate(), Some(0.02)); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - let expected = ClassicHllConfidence::new(1_000, 0.02) - .unwrap() - .failure_probability(17) - .unwrap(); - let bound = guarantee.bind(&|statistic| { - let (precision, eps, source) = parse_classic_failure_statistic(statistic)?; - (source == "m").then(|| classic_failure_probability(precision, eps, 1_000)) - }); - assert_eq!(bound.spec.failure_probability.evaluate(), Some(expected)); - assert!(expected <= 0.01); - for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] { - assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0); - } - // Not sized for a target: its relative standard error. - let unsized_guarantee = classic_guarantee(12, None, "m").unwrap(); - assert_eq!( - unsized_guarantee.spec.error.evaluate(), - Some(relative_standard_error(12)) - ); - assert!(classic_guarantee(3, Some(0.02), "m").is_none()); - } - - /// The largest distinct bound certified at δ = 0.01 for a target ε, by - /// precision (#694's table): past the former 4096 computation cap at - /// precision 18. - #[test] - fn largest_certified_bounds_by_precision() { - for (eps, precision, largest) in [ - (0.05, 15, 1176), - (0.05, 18, 9636), - (0.02, 17, 1842), - (0.02, 18, 3791), - (0.01, 18, 72), - ] { - assert!( - classic_failure_probability(precision, eps, largest) <= 0.01, - "eps={eps} p={precision} n={largest}" - ); - assert!( - classic_failure_probability(precision, eps, largest + 1) > 0.01, - "eps={eps} p={precision} n={}", - largest + 1 - ); - } - } - - /// Exact occupancy probabilities independently check both tails for every N. - #[test] - fn probability_bound_dominates_exact_occupancy_distribution() { - for precision in 4..=10 { - let m = 1usize << precision; - let max_n = 64.min(m / 2); - for eps in [0.05, 0.2, 0.6] { - let model = ClassicHllConfidence::new(max_n as u32, eps).unwrap(); - let Some(bound) = model.failure_probability(precision) else { - continue; - }; - let mut occupancy = vec![0.0; max_n + 1]; - occupancy[0] = 1.0; - for n in 1..=max_n { - let mut next = vec![0.0; max_n + 1]; - for k in 0..n { - next[k] += occupancy[k] * k as f64 / m as f64; - next[k + 1] += occupancy[k] * (m - k) as f64 / m as f64; - } - occupancy = next; - let actual: f64 = occupancy - .iter() - .enumerate() - .filter_map(|(k, &prob)| { - let estimate = -(m as f64) * (-(k as f64) / (m as f64)).ln_1p(); - ((estimate - n as f64).abs() > eps * n as f64).then_some(prob) - }) - .sum(); - assert!( - actual <= bound + 1e-12, - "p={precision} n={n} eps={eps}: {actual}>{bound}" - ); - } - } - } - } - /// The model's evaluation formula matches the actual classic estimator after merge. - #[test] - fn native_classic_estimator_and_merged_registers_use_the_same_contract() { - use asap_sketchlib::sketches::hll::{Classic, HyperLogLogP16}; - let model = ClassicHllConfidence::new(128, 0.05).unwrap(); - assert!(model.failure_probability(16).unwrap() < 0.01); - let mut single = HyperLogLogP16::::new(); - let mut left = HyperLogLogP16::::new(); - let mut right = HyperLogLogP16::::new(); - for n in 0..128u64 { - // SplitMix64 supplies deterministic test hashes, not a proof of randomness. - let mut h = n.wrapping_add(0x9e3779b97f4a7c15); - h = (h ^ (h >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); - h = (h ^ (h >> 27)).wrapping_mul(0x94d049bb133111eb); - h ^= h >> 31; - single.update_with_hash(h); - if n % 2 == 0 { - left.update_with_hash(h); - } else { - right.update_with_hash(h); - } - } - left.merge(&right); - assert_eq!(single.registers_as_slice(), left.registers_as_slice()); - let zeroes = left - .registers_as_slice() - .iter() - .filter(|&&r| r == 0) - .count(); - let expected = (65536.0 * (65536.0 / zeroes as f64).ln()) as usize; - assert_eq!(left.estimate(), expected); - assert!((expected as f64 - 128.0).abs() / 128.0 <= 0.05); - } - #[test] - fn generic_rse_sizing_does_not_certify_confidence() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let c = default_cardinality(); - let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Hll, - ¶ms, - &SketchStatistic::Cardinality, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(meets(&g, 0.01, None)); - assert!(!meets(&g, 0.01, Some(0.01))); - } -} diff --git a/crates/accuracy/src/guarantee/kll.rs b/crates/accuracy/src/guarantee/kll.rs index 0c509da0f..058bcfbc3 100644 --- a/crates/accuracy/src/guarantee/kll.rs +++ b/crates/accuracy/src/guarantee/kll.rs @@ -4,18 +4,22 @@ use super::*; use asap_types::summary_type::KLL_MAX_K; +/// The rank error of `k` at failure probability `failure_probability`: the +/// calibration at 0.01 and above, scaled by [`delta_scale`] below. `None` +/// for a `k` above the library's cap or a probability outside (0, 1]. pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, + failure_probability: f64, ) -> Option { let SketchParams::Kll { k } = params else { return None; }; - if *k > KLL_MAX_K { + if *k > KLL_MAX_K || !(failure_probability > 0.0 && failure_probability <= 1.0) { return None; } - let delta = KLL_CALIBRATED_FAILURE_PROBABILITY; + let delta = failure_probability.min(KLL_CALIBRATED_FAILURE_PROBABILITY); let scale = delta_scale(delta)?; let contract = if delta < KLL_CALIBRATED_FAILURE_PROBABILITY { KLL_DELTA_SCALED_CONTRACT @@ -54,7 +58,7 @@ pub(crate) fn kll_rank_error_99(k: u32) -> f64 { /// It grows faster as `delta` shrinks than the KLL bound's /// `sqrt(ln(1/delta))` (1.277 vs 1.225 at `delta` = 0.001), so it is the /// conservative choice. 1 for `delta >= 0.01`; `None` unless `0 < delta < 1`. -pub(super) fn delta_scale(delta: f64) -> Option { +fn delta_scale(delta: f64) -> Option { if !(delta > 0.0 && delta < 1.0) { None } else if delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { @@ -122,30 +126,6 @@ fn normal_quantile(p: f64) -> f64 { mod tests { use super::*; - #[test] - fn local_guarantee_inverts_rank_sizing() { - use crate::guarantee::sizing_oracle::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(meets(&g, 0.01, None)); - assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(meets(&g, 0.01, Some(0.01))); - assert_eq!(g.approximate_layer_count(), 1); - assert!(g.provenance.iter().any(|source| matches!( - source, - GuaranteeSource::SketchEvaluation { contract, .. } - if contract == "apache_datasketches_kll_empirical_99_a9b42755072b" - ))); - } - /// A hand-built `k` above the library's cap gives no guarantee. #[test] fn a_k_above_the_library_cap_gives_no_guarantee() { @@ -154,6 +134,7 @@ mod tests { &SketchAlgorithm::Kll, &SketchParams::Kll { k }, &SketchStatistic::Quantile { q: 0.99 }, + KLL_CALIBRATED_FAILURE_PROBABILITY, ) }; assert!(g(KLL_MAX_K).is_some()); @@ -164,7 +145,6 @@ mod tests { /// `z(1 − δ/2) / z(1 − 0.01/2)`. #[test] fn delta_scale_is_the_gaussian_quantile_ratio() { - use crate::guarantee::sizing_oracle::{kll_k, kll_params}; assert_eq!(delta_scale(0.01), Some(1.0)); assert_eq!(delta_scale(0.5), Some(1.0)); let ratio = delta_scale(0.001).unwrap(); @@ -179,11 +159,5 @@ mod tests { let scale = delta_scale(delta).unwrap(); assert!((scale - expected).abs() < 1e-8, "{delta}: {scale}"); } - assert_eq!( - kll_params(0.01, 0.001), - SketchParams::Kll { - k: kll_k(0.01 / ratio), - } - ); } } diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs index 60a98efe7..24d070ae1 100644 --- a/crates/accuracy/src/guarantee/mod.rs +++ b/crates/accuracy/src/guarantee/mod.rs @@ -5,28 +5,45 @@ use asap_types::accuracy::{ use asap_types::ir::schema::FieldDataType; use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; -#[cfg(test)] -use crate::requirements::meets; use asap_types::summary_type::SummaryTypeCapability; +use crate::target_limit; + +mod binomial; pub(crate) mod cardinality; pub(crate) mod cms; pub(crate) mod count_sketch; pub(crate) mod ddsketch; -pub mod hll; +pub(crate) mod hll; pub(crate) mod kll; -#[cfg(test)] -mod sizing_oracle; -pub mod univmon; +pub(crate) mod univmon; + +use binomial::binomial_upper_tail; -#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. -pub(super) fn sketch_guarantee( +/// The guarantee of θ for `query`: where one parameter sets both the error +/// and the failure probability (KLL's and KMV's `k`), the one that holds at +/// `failure_probability`; where structural parameters set them apart (a +/// Count-Min's width and depth, HLL's precision), the one θ fixes, whose +/// failure probability may exceed `failure_probability` or be unknown +/// (HLL's). A Hydra state reads each subpopulation's state, so it has +/// that state's guarantee. +fn guarantee_of_parameters( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, + failure_probability: f64, ) -> Option { match params { - SketchParams::Kll { .. } => kll::guarantee(algorithm, params, query), + SketchParams::Kll { .. } => kll::guarantee(algorithm, params, query, failure_probability), + SketchParams::Kmv { .. } => { + cardinality::guarantee(algorithm, params, query, failure_probability) + } + SketchParams::Hydra(hydra) => guarantee_of_parameters( + algorithm, + &hydra.per_subpopulation(), + query, + failure_probability, + ), SketchParams::DDSketch { .. } => ddsketch::guarantee(algorithm, params, query), SketchParams::Hll { .. } => hll::generic_guarantee(algorithm, params, query), SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => { @@ -35,14 +52,29 @@ pub(super) fn sketch_guarantee( SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { count_sketch::guarantee(algorithm, params, query) } - SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), - SketchParams::Hydra(hydra) => { - sketch_guarantee(algorithm, &hydra.per_subpopulation(), query) - } } } +/// The guarantee [`theoretical_guarantee`] gives, before it checks that θ +/// fails at most `failure_probability` rarely: its metric, and whether its +/// failure probability is known at all. `None` when the summary type does +/// not answer `statistic` or has no error model for it. +pub(crate) fn guarantee_before_failure_probability_check( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, +) -> Option { + let FieldDataType::Sketch(algorithm) = summary_type else { + return None; + }; + if !SummaryTypeCapability::of(summary_type).answers(statistic) { + return None; + } + guarantee_of_parameters(algorithm, parameters, statistic, failure_probability) +} + fn bounded_guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, @@ -69,7 +101,11 @@ fn bounded_guarantee( /// `summary_type` with parameters `parameters` at failure probability /// `failure_probability` (the guarantee's own failure probability): the /// algorithm's error formula (concepts doc, section 2, "Where the guarantee -/// comes from"). `None` when this θ cannot fail that rarely, when the +/// comes from"). Where one parameter sets both the error and the failure +/// probability (KLL's and KMV's `k`), the error is the one that holds at +/// `failure_probability`; where structural parameters set them apart (a +/// Count-Min's width and depth), the bound holds only when θ fails at most +/// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch /// readouts without a certified bound). @@ -79,34 +115,121 @@ pub fn theoretical_guarantee( statistic: &SketchStatistic, failure_probability: f64, ) -> Option { - match summary_type { - FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { - let _ = (parameters, failure_probability); - todo!("#1207 PR 4: the algorithm's error formula at θ, at failure_probability") - } - _ => None, - } + guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + ) + .filter(|guarantee| { + guarantee + .spec + .failure_probability + .evaluate() + .is_some_and(|own| own <= target_limit(failure_probability)) + }) } -/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded -/// up by a relative 1e-12 so rounding never certifies a smaller tail. -pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { - if k == 0 || p >= 1.0 { - return 1.0; - } - if k > n || p <= 0.0 { - return 0.0; - } - let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); - let mut ln_choose = 0.0; - let mut tail = 0.0; - for i in 0..=n { - if i > 0 { - ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); - } - if i >= k { - tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::scalar::expr_ir::ColumnRef; + + /// Every listed θ has a theoretical guarantee for the statistic its + /// algorithm is listed for, at failure probability 0.01, in the + /// algorithm's metric; along a list of one structure the error never + /// grows (the lists run weakest first). + #[test] + fn every_listed_parameters_has_a_guarantee_at_one_percent() { + use asap_types::summary_type::parameter_list; + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: Some("1".into()), + }; + for (algorithm, statistic, metric) in [ + ( + SketchAlgorithm::Kll, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Rank, + ), + ( + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Kmv, + SketchStatistic::Cardinality, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Cms, + point.clone(), + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), + ( + SketchAlgorithm::CountSketch, + point, + AccuracyMetric::ABSOLUTE_STREAM_L2, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 10 }, + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), + ] { + let summary_type = FieldDataType::Sketch(algorithm); + let errors: Vec = parameter_list(algorithm) + .iter() + .map(|parameters| { + let guarantee = + theoretical_guarantee(&summary_type, parameters, &statistic, 0.01) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); + guarantee.spec.error.evaluate().unwrap() + }) + .collect(); + assert!(!errors.is_empty(), "{algorithm:?}"); + if matches!( + algorithm, + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch | SketchAlgorithm::Kmv + ) { + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{algorithm:?}: {errors:?}" + ); + } } } - (tail * (1.0 + 1e-12)).min(1.0) + + /// One parameter sets both (KLL's k): its rank error grows as the + /// required failure probability shrinks. Separate parameters (CMS width + /// and depth): the bound holds only when the depth fails rarely enough. + #[test] + fn a_guarantee_is_evaluated_at_the_required_failure_probability() { + let quantile = SketchStatistic::Quantile { q: 0.5 }; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let k = SketchParams::Kll { k: 547 }; + let error_at = |delta| { + theoretical_guarantee(&kll, &k, &quantile, delta) + .unwrap() + .spec + .error + .evaluate() + .unwrap() + }; + assert_eq!(error_at(1.0), error_at(0.01)); + assert!(error_at(1e-6) > error_at(0.01)); + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms); + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let depth = |depth| SketchParams::Cms { width: 272, depth }; + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4).is_some()); + // A statistic the summary type does not answer has no guarantee. + assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01).is_none()); + } } diff --git a/crates/accuracy/src/guarantee/sizing_oracle.rs b/crates/accuracy/src/guarantee/sizing_oracle.rs deleted file mode 100644 index 0df88e3cb..000000000 --- a/crates/accuracy/src/guarantee/sizing_oracle.rs +++ /dev/null @@ -1,676 +0,0 @@ -//! The closed-form sizing per summary type that the predefined parameter -//! lists (`asap_types::summary_type::parameter_list`) are checked against: -//! the parameters of a summary type for error `ε` at failure probability -//! `δ`. Test-only; the planner itself only reads the predefined lists. -use super::*; -use asap_types::ir::operator::AggIntent; -use asap_types::summary_type::KLL_MAX_K; - -/// The parameters of algorithm `kind` for error `eps` at failure -/// probability `delta`. The predefined lists (`parameter_list`) are this -/// sizing over an error grid, which a test keeps equal. -pub(super) fn size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchAlgorithm::UnivMon => univmon_params(intent, eps, delta), - SketchAlgorithm::Kll => kll_params(eps, delta), - SketchAlgorithm::Cms => SketchParams::Cms { - width: cms_width(eps), - depth: cms_depth(delta), - }, - SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll_precision(eps, delta), - }, - SketchAlgorithm::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms_width(eps), - depth: cms_depth(delta), - heap_size: topk_capacity(k, eps), - } - } - // Non-preferred candidates (DDSketch / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: eps }, - SketchAlgorithm::Kmv => SketchParams::Kmv { - k: kmv_k(eps, delta), - }, - SketchAlgorithm::CountSketch => { - let width = count_sketch_width(eps); - SketchParams::CountSketch { - width, - depth: count_sketch_depth(width, delta), - } - } - SketchAlgorithm::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - let width = count_sketch_width(eps); - SketchParams::CountSketchWithHeap { - width, - depth: count_sketch_depth(width, delta), - heap_size: topk_capacity(k, eps), - } - } - // Listed, not `_`: a family `asap_sketchlib` adds fails to compile - // here until the planner sizes it or lists it as not offered. - SketchAlgorithm::CountSketchWithL2Norm - | SketchAlgorithm::CmsWithSpaceSaving - | SketchAlgorithm::CountSketchWithSpaceSaving - | SketchAlgorithm::CmsFold - | SketchAlgorithm::CountSketchFold - | SketchAlgorithm::HllHip - | SketchAlgorithm::KllDynamic - | SketchAlgorithm::SpaceSaving - | SketchAlgorithm::Elastic - | SketchAlgorithm::Coco - | SketchAlgorithm::Bloom - | SketchAlgorithm::SetAggregator - | SketchAlgorithm::UnivMonTwoTier - | SketchAlgorithm::UnivMonQ - | SketchAlgorithm::HydraCms - | SketchAlgorithm::HydraCountSketch - | SketchAlgorithm::HydraKll - | SketchAlgorithm::HydraHll - | SketchAlgorithm::HydraUnivMon => { - unreachable!("Pass 1 offers only the families in ALGORITHM_PREFERENCE, not {kind:?}") - } - } -} - -/// Accuracy-dependent candidate budget, not a completeness theorem. The -/// membership model must still certify the selected set independently. -fn topk_capacity(k: usize, eps: f64) -> u32 { - u32::try_from(k) - .unwrap_or(u32::MAX) - .max(saturating_ceil(1.0 / eps, 1, 1 << 26)) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - -// Count-Min Sketch. - -/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. -fn cms_width(eps: f64) -> u32 { - saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) -} - -/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. -/// δ = 0.01 → depth 5. -fn cms_depth(delta: f64) -> u32 { - saturating_ceil((1.0 / delta).ln(), 1, 32) -} - -// Count Sketch. - -/// CountSketch `L2` point-query width: ε = sqrt(3/w). -pub(super) fn count_sketch_width(eps: f64) -> u32 { - saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) -} - -/// The smallest positive odd depth whose binomial median failure -/// probability at `width` is at most `delta`, capped at 255. -pub(super) fn count_sketch_depth(width: u32, delta: f64) -> u32 { - if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { - return 255; - } - (1..=255) - .step_by(2) - .find(|&depth| { - count_sketch::count_sketch_failure_probability(width, depth) - .is_some_and(|tail| tail <= delta) - }) - .unwrap_or(255) -} - -// KMV. - -/// The largest `k` sizing returns. -const KMV_MAX_K: u32 = 1 << 26; - -/// The smallest `k` whose KMV failure probability at relative error `eps` -/// is at most `delta` ([`cardinality::kmv_failure_probability`]), up to -/// `2^26`; a degenerate `eps` or `delta` saturates to `2^26`. -fn kmv_k(eps: f64, delta: f64) -> u32 { - use cardinality::{kmv_failure_probability, KMV_MIN_K}; - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return KMV_MAX_K; - } - // The overestimate term alone at most `δ` bounds `k` below; both terms - // at most `δ/2` bound it above. - let k_at = |ln_inverse: f64| { - saturating_ceil( - 1.0 + (1.0 + eps) * (2.0 + eps) * ln_inverse / (eps * eps), - KMV_MIN_K, - KMV_MAX_K, - ) - }; - let (mut lo, mut hi) = (k_at((1.0 / delta).ln()), k_at((2.0 / delta).ln())); - if kmv_failure_probability(hi, eps) > delta { - return KMV_MAX_K; - } - while lo < hi { - let mid = lo + (hi - lo) / 2; - if kmv_failure_probability(mid, eps) <= delta { - hi = mid; - } else { - lo = mid + 1; - } - } - hi -} - -// HLL. - -/// Classic HLL's precision for relative error `eps` with failure -/// probability `delta`, without data. Below `1/ε` distinct items a single -/// bucket collision already exceeds the error, and the chance of one is -/// about `N²/(2m)`; a certificate for any bound of at least `1/ε` -/// therefore needs `m ≥ 1/(2ε²δ)`. This is the smallest such precision at -/// which [`hll::ClassicHllConfidence`] certifies `delta` at `eps` for every -/// bound up to `⌈1/ε⌉` (18 when none up to 18 does). -fn hll_precision(eps: f64, delta: f64) -> u8 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return 18; - } - let start = saturating_ceil((1.0 / (2.0 * eps * eps * delta.min(1.0))).log2(), 4, 18) as u8; - let reference = (1.0 / eps) - .ceil() - .min(f64::from(hll::MAX_CONTRACT_DISTINCT)) as u64; - (start..=18) - .find(|&p| hll::classic_failure_probability(p, eps, reference) <= delta) - .unwrap_or(18) -} - -/// The smallest precision, at least `at_least`, whose classic failure -/// probability at `relative_error` over a source of at most `max_distinct` -/// distinct items is at most `failure_probability`; `at_least` when none up -/// to 18 is: [`hll_precision`]'s rule for a source's own distinct bound. -fn min_precision( - relative_error: f64, - failure_probability: f64, - max_distinct: u64, - at_least: u8, -) -> u8 { - (at_least..=18) - .find(|&p| { - hll::classic_failure_probability(p, relative_error, max_distinct) <= failure_probability - }) - .unwrap_or(at_least) -} - -/// The smallest precision at which classic HLL certifies failure -/// probability `delta` at `relative_error` over at most `max_distinct` -/// distinct items ([`hll::ClassicHllConfidence`]); `None` when `delta` is -/// not in (0, 1), the bound or error is outside the contract, or no -/// precision up to 18 certifies it. -fn classic_hll_precision(max_distinct: u32, relative_error: f64, delta: f64) -> Option { - if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { - return None; - } - let model = hll::ClassicHllConfidence::new(max_distinct, relative_error)?; - (4..=hll::MAX_PRECISION).find(|&p| model.failure_probability(p).is_some_and(|d| d <= delta)) -} - -// KLL. - -/// Invert Apache DataSketches' empirical 99th-percentile, single-sided KLL -/// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at -/// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that -/// needs more gets that `k`, whose guarantee then misses the target. -pub(super) fn kll_k(eps: f64) -> u32 { - saturating_ceil( - (kll::KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / kll::KLL_RANK_ERROR_EXPONENT_99), - 8, - KLL_MAX_K, - ) -} - -/// `k` for rank error `eps` at failure probability `delta`: the empirical -/// 99% calibration for `delta >= 0.01`; below, the calibration at -/// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. -pub(super) fn kll_params(eps: f64, delta: f64) -> SketchParams { - if delta.is_nan() || delta <= 0.0 || delta >= kll::KLL_CALIBRATED_FAILURE_PROBABILITY { - return SketchParams::Kll { k: kll_k(eps) }; - } - SketchParams::Kll { - k: kll_k(eps / kll::delta_scale(delta).unwrap_or(1.0)), - } -} - -// UnivMon. - -/// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and -/// seed 19 picks the layers a key reaches. -const UNIVMON_MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; -/// The most rows an L2 sketch is sized with: the deepest odd shape within -/// the 20 rows the executor builds ([`univmon::hash_independent`]), so the -/// row median is one row's estimate, and within the hash budget at the -/// more than 20 columns an L2 guarantee needs (5 column bits: -/// 19 · 6 = 114 ≤ 128). -const UNIVMON_MAX_L2_ROWS: u32 = 19; -/// Each row of a layer reads its column and sign bits from one hash. -const UNIVMON_HASH_BITS: u32 = 128; -/// The largest distinct-count heap sizing returns. -const UNIVMON_MAX_HEAP: u32 = 1 << 26; - -/// The shape baseline for readouts without a size model: the total (exact -/// whatever the shape) and entropy (uncertified). -pub(super) const UNIVMON_BASELINE: SketchParams = SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1024, - layers: 16, -}; - -/// The smallest rows and columns whose L2 guarantee meets `(eps, delta)`, -/// clamped to the hash budget: with more rows than -/// [`UNIVMON_MAX_L2_ROWS`], or more columns than `rows` leave bits for, the -/// shape is the largest one built and its guarantee misses the target. -fn univmon_l2_shape(eps: f64, delta: f64) -> (u32, u32) { - use univmon::{l2_failure_probability, l2_relative_error, L2_ROW_FAILURE}; - let rows = (1..=UNIVMON_MAX_L2_ROWS) - .find(|&rows| l2_failure_probability(rows) <= delta) - .unwrap_or(UNIVMON_MAX_L2_ROWS); - let max_bits = (UNIVMON_HASH_BITS / rows - 1).min(26); - let max_cols = 1u32 << max_bits; - let meets = |cols: u32| l2_relative_error(cols).is_some_and(|error| error <= eps); - if !(eps.is_finite() && eps > 0.0) || !meets(max_cols) { - return (rows, max_cols); - } - // The norm's error `ε` needs `ε₂ ≤ 1 − (1 − ε)²`; a power-of-two width - // needs `2/(0.1 ε₂²)` columns and any other `9/(4 · 0.1 ε₂²)`. - let f2_error = 1.0 - (1.0 - eps.min(1.0)).powi(2); - let general = 9.0 / (4.0 * L2_ROW_FAILURE * f2_error * f2_error); - // Off powers of two the error falls as the width grows, and a power of - // two meets the target whenever its non-power-of-two neighbours do, so - // walking down from the estimate stops at the smallest such width. - let mut general = saturating_ceil(general, 1, max_cols); - while general > 1 && meets(general - 1) { - general -= 1; - } - while !meets(general) { - general += 1; - } - let power_of_two = (0..=max_bits) - .map(|bits| 1u32 << bits) - .find(|&cols| meets(cols)) - .unwrap_or(max_cols); - (rows, general.min(power_of_two)) -} - -/// The Chebyshev share of the distinct count's failure probability at -/// heap `k` and relative error `eps`: `1.35/(k ε²)`. -fn univmon_distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { - univmon::DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) -} - -/// The smallest heap `k ≥ 8` whose distinct count misses by more than -/// `eps · F0` with probability at most `delta`, top-layer completeness -/// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's -/// guarantee misses the target. -fn univmon_distinct_heap(eps: f64, delta: f64) -> u32 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return UNIVMON_MAX_HEAP; - } - let meets = |k: u32| { - univmon_distinct_chebyshev(k, eps) + univmon::incomplete_top_layer_bound(k) <= delta - }; - let mut k = saturating_ceil( - univmon::DISTINCT_VARIANCE / (delta * eps * eps), - univmon::DISTINCT_MIN_HEAP, - UNIVMON_MAX_HEAP, - ); - while !meets(k) && k < UNIVMON_MAX_HEAP { - k += 1; - } - k -} - -/// Size a UnivMon for one readout's `(eps, delta)`: -/// - L2: the smallest rows and columns meeting the target -/// ([`univmon_l2_shape`]), one layer (L2 reads only layer 0) and a -/// one-key heap; -/// - distinct count: the smallest heap meeting it -/// ([`univmon_distinct_heap`]) and the most layers, since the layers the -/// top layer needs depend on the source's distinct bound, which plan -/// selection binds. The bound holds whatever the Count Sketch keeps in -/// the heaps, so each layer's is one counter; -/// - total and entropy: the baseline shape (the total is exact; entropy -/// has no certified bound). -fn univmon_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { - match intent { - AggIntent::FrequencyL2 { .. } => { - let (sketch_rows, sketch_cols) = univmon_l2_shape(eps, delta); - SketchParams::UnivMon { - heap_size: 1, - sketch_rows, - sketch_cols, - layers: 1, - } - } - AggIntent::Cardinality { .. } => SketchParams::UnivMon { - heap_size: univmon_distinct_heap(eps, delta), - sketch_rows: 1, - sketch_cols: 1, - layers: UNIVMON_MAX_LAYERS, - }, - _ => UNIVMON_BASELINE, - } -} - -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; - - /// Classic HLL's precision grows until its failure probability over the - /// source's distinct bound meets the target, up to 18. - #[test] - fn classic_hll_precision_grows_for_its_failure_probability() { - let p = min_precision(0.05, 0.01, 64, 4); - assert!(p > 4); - assert!(hll::classic_failure_probability(p, 0.05, 64) <= 0.01); - assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); - assert_eq!(min_precision(0.05, 1e-30, 4096, 12), 12); - } - - /// A supported estimator contract supplies a probability, unlike - /// generic HLL RSE: sized for δ = 0.01 over at most 128 distinct items, - /// the classic guarantee binds to a known failure probability. - #[test] - fn bounded_classic_hll_has_a_feasible_confidence_target() { - let precision = - classic_hll_precision(128, 0.05, 0.01).expect("finite confidence-sized HLL"); - let guarantee = hll::classic_guarantee(precision, Some(0.05), "m") - .unwrap() - .bind(&|statistic| { - let (precision, eps, _) = hll::parse_classic_failure_statistic(statistic)?; - Some(hll::classic_failure_probability(precision, eps, 128)) - }); - assert!(!guarantee.has_unknown()); - assert!(guarantee.spec.failure_probability.evaluate().unwrap() <= 0.01); - assert_eq!(guarantee.spec.error.evaluate(), Some(0.05)); - } - - /// Tighter confidence must increase precision or explicitly become unavailable. - #[test] - fn sizing_and_domain_limits_are_consistent() { - let model = hll::ClassicHllConfidence::new(128, 0.05).unwrap(); - let precision = |delta| classic_hll_precision(128, 0.05, delta); - assert!(precision(0.001).unwrap() > precision(0.01).unwrap()); - assert!(precision(1e-12).is_none()); - assert!(precision(0.0).is_none()); - assert!(precision(f64::NAN).is_none()); - assert!(model.failure_probability(3).is_none()); - assert!(model.failure_probability(19).is_none()); - assert!(model.failure_probability(7).is_none()); - for (n, e) in [ - (0, 0.05), - (hll::MAX_CONTRACT_DISTINCT + 1, 0.05), - (128, 0.0), - (128, f64::NAN), - ] { - assert!(hll::ClassicHllConfidence::new(n, e).is_none()); - } - } - - /// Each algorithm's predefined list (`parameter_list`) is this sizing - /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, - /// smallest first, without repeats; KMV's and UnivMon's distinct counts - /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, - /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant - /// each of those with - /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also - /// combines each L2 shape with each distinct-count shape (28 entries). - #[test] - fn parameter_list_is_the_sizing_over_the_error_grid() { - use asap_types::summary_type::parameter_list; - let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; - let distinct_grid = [0.1, 0.05, 0.02, 0.01]; - let cms_depths = [5, 7, 10, 12]; - let count_sketch_depths = [47, 81, 117, 155]; - let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; - let l2 = AggIntent::FrequencyL2 { col: None }; - let top_1 = AggIntent::TopK { k: 1 }; - let quantile = default_quantile(0.5); - let distinct = default_cardinality(); - for (algorithm, intents) in [ - (SketchAlgorithm::Kll, vec![&quantile]), - (SketchAlgorithm::DDSketch, vec![&quantile]), - (SketchAlgorithm::Hll, vec![&distinct]), - (SketchAlgorithm::Kmv, vec![&distinct]), - (SketchAlgorithm::Cms, vec![&distinct]), - (SketchAlgorithm::CountSketch, vec![&distinct]), - (SketchAlgorithm::CmsWithHeap, vec![&top_1]), - (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), - (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), - ] { - let mut sized: Vec = Vec::new(); - for intent in intents { - let grid: &[f64] = match (algorithm, intent) { - ( - SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, - AggIntent::Cardinality { .. }, - ) => &distinct_grid, - _ => &full_grid, - }; - for &eps in grid { - let listed: Vec = match size_params(algorithm, intent, eps, 0.01) - { - SketchParams::Cms { width, .. } => cms_depths - .iter() - .map(|&depth| SketchParams::Cms { width, depth }) - .collect(), - SketchParams::CountSketch { width, .. } => count_sketch_depths - .iter() - .map(|&depth| SketchParams::CountSketch { width, depth }) - .collect(), - SketchParams::CmsWithHeap { width, .. } => cms_depths - .iter() - .flat_map(|&depth| { - heap_sizes - .iter() - .map(move |&heap_size| SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }) - }) - .collect(), - SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths - .iter() - .flat_map(|&depth| { - heap_sizes.iter().map(move |&heap_size| { - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } - }) - }) - .collect(), - other => vec![other], - }; - for params in listed { - if !sized.contains(¶ms) { - sized.push(params); - } - } - } - } - let list = parameter_list(algorithm); - let (listed_sized, combined): (Vec<_>, Vec<_>) = - list.iter().partition(|params| sized.contains(params)); - assert_eq!( - format!("{listed_sized:?}"), - format!("{:?}", sized.iter().collect::>()), - "{algorithm:?}" - ); - let expected_combined = if algorithm == SketchAlgorithm::UnivMon { - 28 - } else { - 0 - }; - assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); - } - } - - /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count; - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - } // ⌈e/0.001⌉, ⌈ln 1000⌉ - ); - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), - SketchParams::Cms { - width: 2719, - depth: 5 - } - ); - assert_eq!( - size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), - SketchParams::Cms { - width: 272, - depth: 5 - } - ); - } - - /// A top-k heap holds `max(k, ⌈1/ε⌉)` entries over a CMS sized as for counts. - #[test] - fn topk_heap_capacity_respects_accuracy_and_output_count() { - let intent = AggIntent::TopK { k: 25 }; - assert_eq!( - size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.01, 0.01), - SketchParams::CmsWithHeap { - width: 272, // ⌈e/0.01⌉ - depth: 5, - heap_size: 100, - } - ); - } - - /// The executor's `CountSketch<_, RegularPath>` takes a column as the - /// low 32 hash bits modulo the width, so a row fails with probability - /// `1/3` (to within `2^-32`) at any width, not `3/8` off powers of two: - /// ε = 0.05 (width 1200) needs depth 47 at δ = 0.01, 81 at 1e-3 and - /// 117 at 1e-4, not 85, 149 and 215. - #[test] - fn depth_follows_the_libraries_modulo_column_hash() { - let width = count_sketch_width(0.05); - assert_eq!(width, 1200); - assert!((count_sketch::row_failure_probability(width) - 1.0 / 3.0).abs() < 1e-6); - assert_eq!( - [0.01, 1e-3, 1e-4].map(|delta| count_sketch_depth(width, delta)), - [47, 81, 117] - ); - } - - /// The depth is the smallest odd one whose binomial tail meets δ. Each - /// is at most Hoeffding's `ln(1/δ) / (2 (1/2 − p)²)` at the row's - /// failure probability `p ≈ 1/3`: `18 ln(1/δ)` at any width. - #[test] - fn count_sketch_depth_is_the_smallest_meeting_the_binomial_tail() { - use count_sketch::count_sketch_failure_probability; - for width in [1024, 3000, 30_000] { - let hoeffding = 18.0 * (1.0 + 1e-4); - for delta in [0.1, 0.01, 0.001] { - let depth = count_sketch_depth(width, delta); - assert!(depth % 2 == 1); - assert!(count_sketch_failure_probability(width, depth).unwrap() <= delta); - if depth > 1 { - assert!(count_sketch_failure_probability(width, depth - 2).unwrap() > delta); - } - assert!( - f64::from(depth) <= hoeffding * (1.0 / delta).ln(), - "{width} {delta}" - ); - } - } - // Any width fails a row with probability 1/3 (to within 2^-32 w/3): - // P(Bin(3, 1/3) ≥ 2) = 7/27. - for width in [1024, 1000] { - let tail = count_sketch_failure_probability(width, 3).unwrap(); - assert!((tail - 7.0 / 27.0).abs() < 1e-6, "{width}"); - } - assert_eq!(count_sketch_failure_probability(1000, 4), None); - } - - /// Pass 1 sizes classic HLL from (ε, δ) without data: the smallest - /// precision with `m ≥ 1/(2ε²δ)` that the tail formula certifies at the - /// target ε, which its guarantee states as its error bound. - #[test] - fn precision_is_sized_from_epsilon_and_delta() { - for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { - let params = size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); - let SketchParams::Hll { precision: sized } = params else { - panic!("{params:?}") - }; - assert_eq!(sized, precision); - } - } - - /// `k` never exceeds the 26 602 `asap_sketchlib` silently caps it at, so - /// a guarantee never claims a larger `k` than runs. - #[test] - fn kll_k_is_capped_at_the_library_maximum() { - assert_eq!(kll_k(1e-6), KLL_MAX_K); - assert_eq!(KLL_MAX_K, 26_602); - } - - /// KLL `k` inverts the rank-error fit: ε = 0.01 → 269, ε = 0.05 → 52. - #[test] - fn accuracy_target_drives_kll_k() { - let q = default_quantile(0.99); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), - SketchParams::Kll { k: 269 } - ); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.05, 0.01), - SketchParams::Kll { k: 52 } - ); - } - - /// A zero ε saturates KLL `k` to the tightest buildable value. - #[test] - fn degenerate_epsilon_saturates_to_tightest_kll_params() { - assert_eq!( - size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), - SketchParams::Kll { k: 26_602 } - ); - } - - /// Example 2's Q1 (ε = 0.02, δ = 0.01) needs a heap of 337,500 keys, - /// `⌈1.35/(δ ε²)⌉`: the top layer's term is negligible at that heap. - /// At (0.5, 0.5) it is not: 1.35/(12 · 0.25) + (e/4)^6 > 0.5, while 13 - /// keys give 0.415 + 0.081. - #[test] - fn distinct_heap_for_example2_q1() { - assert_eq!(univmon_distinct_heap(0.02, 0.01), 337_500); - assert_eq!(univmon_distinct_heap(0.5, 0.5), 13); - } -} diff --git a/crates/accuracy/src/guarantee/univmon.rs b/crates/accuracy/src/guarantee/univmon.rs index 55ec1bf58..5973dcfed 100644 --- a/crates/accuracy/src/guarantee/univmon.rs +++ b/crates/accuracy/src/guarantee/univmon.rs @@ -6,10 +6,8 @@ //! - L2 reads layer 0's Count Sketch directly: per row, Chebyshev with //! `Var ≤ 2q F2²`, and the median of the rows fails only when at least //! `⌈r/2⌉` rows fail, an exact binomial tail; -//! - the distinct count is unbiased with `Var ≤ 1.35 F0²/k` for a heap of -//! `k ≥ 8` once the top layer is complete, which needs the source's -//! distinct bound, so its guarantee is built where the source is known -//! (`distinct_guarantee`) and plan selection binds it; +//! - the distinct count has no guarantee here: it needs the source's +//! distinct bound; //! - entropy has no certified bound. //! //! Every bound assumes the key hash is uniformly random and the shape keeps @@ -21,19 +19,9 @@ use super::*; /// The per-row failure probability the L2 guarantee fixes: a row misses by /// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it /// lets a shape state one `(ε, δ)`: the columns give `ε`, the rows `δ`. -pub(super) const L2_ROW_FAILURE: f64 = 0.1; -/// `Var[estimate_cardinality] ≤ 1.35 F0²/k` for `k ≥ 8`, the constant the library -/// proves (it also states the looser `2 F0²/k`). -pub(super) const DISTINCT_VARIANCE: f64 = 1.35; -/// The smallest heap the distinct-count variance bound holds for. -pub(super) const DISTINCT_MIN_HEAP: u32 = 8; +const L2_ROW_FAILURE: f64 = 0.1; const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; -const DISTINCT_CONTRACT: &str = "univmon_distinct_complete_top_layer_chebyshev_v1"; -/// Names the probability that the top layer is incomplete, left symbolic -/// in `max_distinct[source]` ([`incomplete_top_layer_statistic`]). -const INCOMPLETE_TOP_LAYER: &str = "univmon_incomplete_top_layer_probability"; -const MAX_DISTINCT: &str = "max_distinct"; pub(super) fn guarantee( algorithm: &SketchAlgorithm, @@ -66,8 +54,8 @@ pub(super) fn guarantee( L2_CONTRACT, )) } - // The distinct count needs the source's distinct bound - // ([`distinct_guarantee`]); entropy has no certified bound. + // The distinct count needs the source's distinct bound; entropy has + // no certified bound. _ => None, } } @@ -94,7 +82,7 @@ pub(crate) fn hash_independent(rows: u32, cols: u32, layers: u8) -> bool { /// [`L2_ROW_FAILURE`] gives `ε₂ = √(2q/0.1)`. The norm then misses by at /// most `1 − √(1 − ε₂)` (the low side; the high side `√(1 + ε₂) − 1` is /// smaller). `None` when `ε₂ ≥ 1`. -pub(super) fn l2_relative_error(cols: u32) -> Option { +fn l2_relative_error(cols: u32) -> Option { let f2_error = (2.0 * column_collision(cols) / L2_ROW_FAILURE).sqrt(); (f2_error < 1.0).then(|| 1.0 - (1.0 - f2_error).sqrt()) } @@ -117,126 +105,25 @@ fn column_collision(cols: u32) -> f64 { /// The median of `rows` independent rows misses only when at least /// `⌈rows/2⌉` of them miss: `P(Binomial(rows, 0.1) ≥ ⌈rows/2⌉)`. -pub(super) fn l2_failure_probability(rows: u32) -> f64 { +fn l2_failure_probability(rows: u32) -> f64 { super::binomial_upper_tail(rows, L2_ROW_FAILURE, rows.div_ceil(2)) } -/// `(e/4)^(k/2)`: the chance the top layer of a heap-`k` pyramid misses a -/// key, once the expected keys reaching it are at most `k/2` (Chernoff). -pub(super) fn incomplete_top_layer_bound(heap_size: u32) -> f64 { - (f64::from(heap_size) / 2.0 * (1.0 - 2.0 * std::f64::consts::LN_2)).exp() -} - -/// The statistic a UnivMon distinct count over `source` leaves its -/// top-layer failure probability as. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { - format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") -} - -/// The heap, layers and source an `incomplete_top_layer_statistic` names. -pub fn parse_incomplete_top_layer_statistic(statistic: &str) -> Option<(u32, u8, &str)> { - let rest = statistic - .strip_prefix(INCOMPLETE_TOP_LAYER)? - .strip_prefix("[k=")?; - let (heap_size, rest) = rest.split_once(",layers=")?; - let (layers, rest) = rest.split_once(']')?; - let source = rest - .strip_prefix('(')? - .strip_prefix(MAX_DISTINCT)? - .strip_prefix('[')? - .strip_suffix("])")?; - Some((heap_size.parse().ok()?, layers.parse().ok()?, source)) -} - -/// The probability an `incomplete_top_layer_statistic` names, given the -/// source's distinct bound `F0_max`: -/// - 0 when `F0_max ≤ k`: the top layer cannot receive more keys than its -/// heap holds; -/// - `(e/4)^(k/2)` when `layers ≥ ⌈log2(F0_max/k)⌉ + 2`, i.e. -/// `F0_max ≤ k · 2^(layers−2)`, so at most `k/2` keys reach the top layer -/// in expectation; -/// - 1 (no certificate) otherwise: the pyramid is too shallow. -pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct: u64) -> f64 { - let (k, max_distinct) = (u128::from(heap_size), u128::from(max_distinct)); - if max_distinct <= k { - 0.0 - } else if layers >= 2 && max_distinct <= k << (layers - 2) { - incomplete_top_layer_bound(heap_size) - } else { - 1.0 - } -} - -/// A UnivMon distinct count over `source`'s own rows, at failure -/// probability `delta`: relative error at most `√(1.35/(k δ_c))`, where -/// `δ_c` is the share of `delta` left after the top layer's, failing with -/// probability `δ_c` plus the chance of an incomplete top layer, left -/// symbolic in the source's distinct bound. `None` for a heap under 8, a -/// `delta` too small for the heap, or a shape the library would not build. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn distinct_guarantee( - params: &SketchParams, - delta: f64, - source: &str, -) -> Option { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } = params - else { - return None; - }; - if *heap_size < DISTINCT_MIN_HEAP - || !hash_independent(*sketch_rows, *sketch_cols, *layers) - || !(delta > 0.0 && delta < 1.0) - { - return None; - } - let chebyshev = delta - incomplete_top_layer_bound(*heap_size); - if chebyshev <= 0.0 { - return None; - } - let relative_error = (DISTINCT_VARIANCE / (f64::from(*heap_size) * chebyshev)).sqrt(); - let statistic = incomplete_top_layer_statistic(*heap_size, *layers, source); - Some(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: relative_error, - }, - ProbabilityExpr::UnionBound { - terms: vec![ - ProbabilityExpr::Constant { value: chebyshev }, - ProbabilityExpr::Unknown { statistic }, - ], - }, - vec![GuaranteeSource::SketchEvaluation { - algorithm: "UnivMon".into(), - contract: DISTINCT_CONTRACT.into(), - params: serde_json::json!({"heap_size": heap_size, "layers": layers, - "max_distinct": format!("{MAX_DISTINCT}[{source}]"), - "relative_error": relative_error, - "variance": "1.35 F0^2 / k", - "hash_assumption": "independent_uniform_layer_hashes", - "mode": "standard", - "population_scope": "complete_evaluation_including_merged_tumbling_windows"}), - query: "Cardinality".into(), - }], - )) -} - #[cfg(test)] mod tests { use super::*; - use crate::guarantee::sizing_oracle::{size_params, UNIVMON_BASELINE}; - use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::operator::AggIntent; - fn l2_intent() -> AggIntent { - AggIntent::FrequencyL2 { col: None } - } + /// Layers `asap_sketchlib` builds. + const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; + + /// A shape for readouts without a size model: the total (exact + /// whatever the shape) and entropy (uncertified). + const BASELINE: SketchParams = SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1024, + layers: 16, + }; fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { SketchParams::UnivMon { @@ -255,55 +142,6 @@ mod tests { ) } - fn with(params: &SketchParams, f: impl FnOnce(&mut u32, &mut u32, &mut u32)) -> SketchParams { - let mut params = params.clone(); - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - .. - } = &mut params - else { - unreachable!() - }; - f(heap_size, sketch_rows, sketch_cols); - params - } - - const TARGETS: [(f64, f64); 6] = [ - (0.01, 0.01), - (0.02, 0.01), - (0.05, 0.01), - (0.05, 0.001), - (0.1, 0.05), - (0.2, 0.2), - ]; - - /// Each L2 shape meets its target as a relative error, and one fewer - /// row or column does not. - #[test] - fn l2_shape_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(SketchAlgorithm::UnivMon, &l2_intent(), eps, delta); - let guarantee = l2(¶ms).expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let fewer_rows = with(¶ms, |_, rows, _| *rows -= 1); - assert!( - l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_rows:?}" - ); - let fewer_cols = with(¶ms, |_, _, cols| *cols -= 1); - assert!( - l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_cols:?}" - ); - } - } - /// The general collision bound `9/(8w)` applies off powers of two: a /// width just past a power of two is worse than that power of two. #[test] @@ -325,37 +163,6 @@ mod tests { assert!(l2_failure_probability(4) > l2_failure_probability(3)); } - /// Shapes past the hash budget or the layer limit certify nothing, and - /// L2 sizing never builds one: a target it cannot reach gets the - /// largest shape it can build, whose guarantee misses it. - #[test] - fn l2_shapes_stay_within_the_hash_budget() { - // 8 · (16 + 1) = 136 > 128. - assert!(l2(&univmon(1, 8, 1 << 16, 1)).is_none()); - assert!(l2(&univmon(1, 7, 1 << 16, 1)).is_some()); - assert!(l2(&univmon(1, 5, 1024, 20)).is_none()); - for (eps, delta) in [(0.001, 1e-6), (1e-5, 0.01), (0.01, 1e-12)] { - let params = size_params(SketchAlgorithm::UnivMon, &l2_intent(), eps, delta); - let SketchParams::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - unreachable!() - }; - assert!( - hash_independent(sketch_rows, sketch_cols, layers), - "{params:?}" - ); - assert!( - !meets(&l2(¶ms).unwrap(), eps, Some(delta)), - "{params:?}" - ); - } - } - /// The executor builds at most 20 rows, the rows a UnivMon state /// encodes, so a 21-row UnivMon certifies nothing even though its hashes /// fit the budget (21 · (5 + 1) = 126 ≤ 128 at 32 columns). @@ -367,99 +174,10 @@ mod tests { assert!(l2(&univmon(1, 21, 32, 1)).is_none()); } - /// Each distinct-count heap meets its target once the source's bound - /// is bound, and one fewer key does not. - #[test] - fn distinct_heap_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta); - let bind = |params: &SketchParams, max_distinct| { - let guarantee = - distinct_guarantee(params, delta, "m").expect("a distinct guarantee"); - guarantee.bind(&|statistic| { - let (k, layers, source) = parse_incomplete_top_layer_statistic(statistic)?; - (source == "m") - .then(|| incomplete_top_layer_probability(k, layers, max_distinct)) - }) - }; - for max_distinct in [9, 10_000_000] { - let guarantee = bind(¶ms, max_distinct); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let smaller = with(¶ms, |heap, _, _| *heap -= 1); - assert!( - !meets(&bind(&smaller, max_distinct), eps, Some(delta)), - "({eps}, {delta}): {smaller:?}" - ); - } - } - } - - /// Without the source's bound the failure probability stays unknown - /// and no target is met; a bound the pyramid is too shallow for gives - /// probability 1. - #[test] - fn distinct_guarantee_needs_the_source_bound_and_enough_layers() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(!meets(&guarantee, 0.05, Some(0.01))); - let SketchParams::UnivMon { heap_size: k, .. } = params else { - unreachable!() - }; - // At most k keys: complete for certain. - assert_eq!(incomplete_top_layer_probability(k, 2, u64::from(k)), 0.0); - // F0_max = k · 2^(L−2) is the deepest bound L layers cover. - let deepest = u64::from(k) << 17; - assert!(incomplete_top_layer_probability(k, 19, deepest) < 1e-100); - assert_eq!(incomplete_top_layer_probability(k, 19, deepest + 1), 1.0); - assert_eq!(incomplete_top_layer_probability(k, 18, deepest), 1.0); - assert_eq!( - incomplete_top_layer_probability(k, 1, u64::from(k) + 1), - 1.0 - ); - assert_eq!(incomplete_top_layer_probability(8, 19, u64::MAX), 1.0); - } - - /// Only a heap of at least 8 and a buildable shape gets a distinct - /// guarantee; the guarantee without a source is none. - #[test] - fn distinct_guarantee_needs_a_heap_of_eight_and_a_buildable_shape() { - assert!(distinct_guarantee(&univmon(7, 1, 1, 19), 0.5, "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 20), 0.5, "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 19), 0.5, "m").is_some()); - let sized = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - assert!(guarantee( - &SketchAlgorithm::UnivMon, - &sized, - &SketchStatistic::Cardinality - ) - .is_none()); - } - - /// The statistic names its heap, layers and source, any source name. - #[test] - fn incomplete_top_layer_statistics_round_trip() { - for source in ["flows_src_ip", "a[b]", "x])(", ""] { - let statistic = incomplete_top_layer_statistic(337_500, 19, source); - assert_eq!( - parse_incomplete_top_layer_statistic(&statistic), - Some((337_500, 19, source)) - ); - } - assert_eq!( - parse_incomplete_top_layer_statistic("max_distinct[m]"), - None - ); - } - /// The total stays exact; entropy stays uncertified. #[test] fn total_is_exact_and_entropy_uncertified() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); + let params = univmon(54_000, 1, 1, MAX_LAYERS); let total = guarantee( &SketchAlgorithm::UnivMon, ¶ms, @@ -472,7 +190,7 @@ mod tests { assert!(total.is_exact()); assert!(guarantee( &SketchAlgorithm::UnivMon, - &UNIVMON_BASELINE, + &BASELINE, &SketchStatistic::FrequencyEntropy ) .is_none()); diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 13be59fc7..230e2b385 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,37 +1,31 @@ //! `asap-accuracy` — the accuracy rules the planning stages share: the -//! planner's built-in analytical rules (estimators, composition, guarantee -//! propagation through a DAG, sizing, budget split and a target's comparison -//! limit [`target_limit`]), the deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ -//! of a logical edge ([`producer_satisfies_logical_requirements`]). -//! -//! Estimator models derive theoretical guarantees, composition propagates -//! them, `propagation` composes them through every node of a DAG, `sizing` -//! sizes a summary for its target, the budget split sizes each approximate -//! input of a composition for an equal share of its target, and -//! `propagation_stats` holds the scoped contracts and statistics the rules -//! consult. `guarantee` derives each summary type's theoretical guarantee, -//! `evidence` reads the deployment's measured accuracy, `requirements` -//! decides $\alpha^{\log}$ and `error_propagation` bounds a consumer's -//! output error by its inputs'. +//! theoretical guarantee of reading a statistic from a summary with given +//! parameters ([`theoretical_guarantee`], from the per-algorithm +//! guarantees in `guarantee`), the deployment's measured accuracy +//! ([`AccuracyEvidence`], in `evidence`), each operator's error contract +//! ([`error_propagation`], in `error_propagation`), the metric of a +//! readout's error ([`readout_metric`], in `readout`), the metric rule +//! ([`accepted_metrics`], in `metric_rule`), each readout's required +//! accuracy ([`readout_levels`], in `readout_caps`), and $\alpha^{\log}$ of +//! a logical edge ([`producer_satisfies_logical_requirements`], in +//! `logical_edge`, with a requirement's comparison limit [`target_limit`]). //! See `docs/design_docs/concepts/accuracy-models.md` for the design. //! //! It depends only on `asap-types`, never on a stage crate, so every stage //! can use it. -mod budget; -pub mod composition; mod error_propagation; mod evidence; -mod function_rules; -pub mod guarantee; -pub mod propagation; -mod propagation_stats; -mod requirements; -pub mod sizing; +mod guarantee; +mod logical_edge; +mod metric_rule; +mod readout; +mod readout_caps; -pub use budget::per_input_target; -pub use error_propagation::{error_propagation, ErrorPropagation}; +pub use error_propagation::{error_propagation, metrics_carried, ErrorPropagation}; pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; pub use guarantee::theoretical_guarantee; -pub(crate) use propagation_stats::PropagationStats; -pub use requirements::{producer_satisfies_logical_requirements, target_limit}; +pub use logical_edge::{producer_satisfies_logical_requirements, target_limit}; +pub use metric_rule::{accepted_metrics, metrics_accepted_on_inputs}; +pub use readout::{readout_metric, readout_metric_of_candidate}; +pub use readout_caps::readout_levels; diff --git a/crates/accuracy/src/logical_edge.rs b/crates/accuracy/src/logical_edge.rs new file mode 100644 index 000000000..583062bff --- /dev/null +++ b/crates/accuracy/src/logical_edge.rs @@ -0,0 +1,320 @@ +//! $\alpha^{\log}$ of a logical edge: whether a producer meets a consumer's +//! logical requirements, and the comparison of an evaluated bound with a +//! required limit ([`target_limit`]). + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; +use asap_types::summary_type::{MergedError, SketchParams, SketchStatistic, SummaryTypeCapability}; + +use crate::guarantee::guarantee_before_failure_probability_check; +use crate::readout::sketch_readout; +use crate::{theoretical_guarantee, AccuracyEvidence}; + +/// Small relative tolerance for comparing an evaluated bound against a +/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected +/// by floating-point noise. +const SATISFACTION_TOLERANCE: f64 = 1e-9; + +/// The largest evaluated error (or failure probability) that meets a +/// required `limit`: `limit` widened by [`SATISFACTION_TOLERANCE`], so a +/// bound equal to it up to floating-point noise meets it. +pub fn target_limit(limit: f64) -> f64 { + limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON +} + +/// $\alpha^{\log}$ of one edge (concepts doc, section 2): `Ok` when +/// `producer` may feed input `input_index_of_consumer` of `consumer`, else +/// why not, for Pass 2's pruned list. `readout_metric_of_candidate` is +/// [`readout_metric_of_candidate`](crate::readout_metric_of_candidate) of +/// `memo`, and `metrics_accepted_on_inputs` is +/// [`metrics_accepted_on_inputs`](crate::metrics_accepted_on_inputs) of +/// `consumer`. +/// - A readout's input (`SummaryEstimate`) from a sketch state: the +/// readout's metric is one its output accepts (the metric rule; no rule +/// where no query with a target reaches it; `NoGuarantee` when, under +/// such a query, the readout has no metric or its guarantee's failure +/// probability is unknown, as HLL's), and the guarantee of the +/// producer's θ for the readout's statistic, at the required failure +/// probability, is within the required error ([`target_limit`]) and in +/// its metric: the deployment's measurement (`evidence`) when it has +/// one, else [`theoretical_guarantee`]; a merged state (any producer but +/// a build) must keep the guarantee (`MergedError::Exact` or +/// `AsIfBuiltOverUnion`). A top-k's heap holds its `k` whatever the +/// requirement. +/// - Any other input: the metric rule, when the producer is a sketch +/// readout: the consumer accepts the readout's metric on that input. +/// Only readouts introduce error, so an edge from any other producer +/// holds; a consumer reading an approximate value through it is cut +/// below, on the readout's edge. +pub fn producer_satisfies_logical_requirements( + memo: &Memo, + readout_metric_of_candidate: &[Vec>], + metrics_accepted_on_inputs: Option<&[AccuracyMetric]>, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input_index_of_consumer: usize, + evidence: &dyn AccuracyEvidence, +) -> Result<(), RootPruneReason> { + let consumer_candidate = memo.candidate(consumer); + let input = input_index_of_consumer; + let readout_metric = |candidate: CandidateForMemoGroupId| { + readout_metric_of_candidate[candidate.group.0 as usize][candidate.index_in_group] + }; + let metric_rule = |producer_metric: AccuracyMetric, accepted_metrics: &[AccuracyMetric]| { + if accepted_metrics.contains(&producer_metric) { + Ok(()) + } else { + Err(RootPruneReason::MetricRule { + consumer, + input, + producer_metric, + accepted_metrics: accepted_metrics.to_vec(), + }) + } + }; + let Operator::ASAP(ASAPOp::SummaryEstimate { .. }) = consumer_candidate.expression.operator + else { + let (Some(producer_metric), Some(accepted)) = + (readout_metric(producer), metrics_accepted_on_inputs) + else { + return Ok(()); + }; + return metric_rule(producer_metric, accepted); + }; + let producer_candidate = memo.candidate(producer); + let (Some((summary_type, statistic)), Some(parameters)) = ( + sketch_readout(memo, consumer_candidate), + producer_candidate.logical_properties.parameters(), + ) else { + // An exact state reads exactly. + return Ok(()); + }; + let summary_type = &summary_type; + if let (SketchStatistic::TopK { k }, Some(heap_size)) = (statistic, heap_size(parameters)) { + if (heap_size as usize) < *k { + return Err(RootPruneReason::HeapSmallerThanTopK { + consumer, + input, + k: *k, + }); + } + } + let required = consumer_candidate.input_requirements[input].required_accuracy; + let failure_probability = required.map_or(1.0, |required| required.failure_probability); + let measured = + evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); + if let Some(accepted) = metrics_accepted_on_inputs { + // A query with a target reads this readout: its error needs a + // metric and a failure probability a guarantee states. + let no_guarantee = RootPruneReason::NoGuarantee { consumer, input }; + let metric = readout_metric(consumer).ok_or(no_guarantee.clone())?; + metric_rule(metric, accepted)?; + let evaluable = measured.is_some() + || guarantee_before_failure_probability_check( + summary_type, + parameters, + statistic, + failure_probability, + ) + .is_some_and(|guarantee| guarantee.spec.failure_probability.evaluate().is_some()); + if !evaluable { + return Err(no_guarantee); + } + } + let Some(required) = required else { + return Ok(()); + }; + let merged = !matches!( + producer_candidate.expression.operator, + Operator::ASAP(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) + ); + let keeps = !merged + || matches!( + SummaryTypeCapability::of(summary_type).merged_error(statistic), + Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) + ); + let guarantee = measured.or_else(|| { + theoretical_guarantee( + summary_type, + parameters, + statistic, + required.failure_probability, + ) + }); + let meets = guarantee.is_some_and(|guarantee| { + guarantee.spec.accuracy_metric == required.metric + && guarantee + .spec + .error + .evaluate() + .is_some_and(|error| error <= target_limit(required.max_error)) + }); + if keeps && meets { + Ok(()) + } else { + Err(RootPruneReason::BelowRequiredAccuracy { + consumer, + input, + required, + }) + } +} + +/// A heap variant's heap size. +fn heap_size(parameters: &SketchParams) -> Option { + match parameters { + SketchParams::CmsWithHeap { heap_size, .. } + | SketchParams::CountSketchWithHeap { heap_size, .. } => Some(*heap_size), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{readout_metric_of_candidate, NoMeasuredGuarantees}; + use asap_types::ir::operator::arguments::Reduction; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::FieldDataType; + use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; + use asap_types::summary_type::{parameter_list, SketchAlgorithm, SummaryUpdate}; + use asap_types::test_support::memo_fixtures::scan; + use asap_types::workload::DataWorkload; + + /// A bound equal to the limit up to floating-point noise meets it; a + /// bound one part in a million above it does not. + #[test] + fn target_limit_admits_only_floating_point_noise() { + assert!(0.01 * (1.0 + 1e-12) <= target_limit(0.01)); + assert!(0.01 * (1.0 + 1e-6) > target_limit(0.01)); + assert!(0.0 <= target_limit(0.0)); + } + + /// A distinct count read from HLL (relative error, failure probability + /// unknown) or UnivMon (no guarantee for it) under a query with a + /// relative target: the edge fails with `NoGuarantee`, its error would + /// reach the root unbounded; with no target reaching it, it holds. + #[test] + fn a_distinct_count_readout_without_an_evaluable_guarantee_serves_no_query_with_a_target() { + for algorithm in [SketchAlgorithm::Hll, SketchAlgorithm::UnivMon] { + let mut memo = Memo::new(&DataWorkload::default()); + let events = memo.find_or_add_group(scan("events", &[], vec![])).unwrap(); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: events, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let state = memo.find_or_add_group(build.clone()).unwrap(); + let parameters = parameter_list(algorithm)[0].clone(); + let producer = memo + .add_candidate( + state, + CandidateForMemoGroup::new( + build, + Some(parameters), + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let readout = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Cardinality, + }); + let distinct = memo.find_or_add_group(readout.clone()).unwrap(); + let consumer = memo + .add_candidate( + distinct, + CandidateForMemoGroup::new( + readout, + None, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) + .unwrap(); + let readout_metrics = readout_metric_of_candidate(&memo); + let edge = |accepted: Option<&[AccuracyMetric]>| { + producer_satisfies_logical_requirements( + &memo, + &readout_metrics, + accepted, + producer, + consumer, + 0, + &NoMeasuredGuarantees, + ) + }; + assert!(edge(None).is_ok(), "{algorithm:?}"); + let relative = Some(&[AccuracyMetric::Relative][..]); + assert!( + matches!( + edge(relative), + Err(RootPruneReason::NoGuarantee { input: 0, .. }) + ), + "{algorithm:?}: {:?}", + edge(relative) + ); + } + } + + /// A top-k heap smaller than k never serves the readout, whatever + /// accuracy is required: the edge from each build into `topk(50)`'s + /// readout fails with `HeapSmallerThanTopK` exactly for the heaps + /// below 50 (sizes 10 and 20, with each width and depth). + #[test] + fn a_heap_smaller_than_k_fails_the_topk_readouts_edge() { + use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, scan_group, sketch_state, + }; + use asap_types::test_support::memo_fixtures::range; + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let state = sketch_state( + &mut memo, + window, + SketchAlgorithm::CmsWithHeap, + Reduction::by(vec![]), + ); + let readout = estimate(&mut memo, state, SketchStatistic::TopK { k: 50 }); + let readout_metrics = readout_metric_of_candidate(&memo); + let mut small = 0; + for index_in_group in 0..memo.groups()[state.0 as usize].candidates.len() { + let build = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + let Some(SketchParams::CmsWithHeap { heap_size, .. }) = + memo.candidate(build).logical_properties.parameters() + else { + panic!("a heap build"); + }; + let edge = producer_satisfies_logical_requirements( + &memo, + &readout_metrics, + None, + build, + own_candidate(readout), + 0, + &NoMeasuredGuarantees, + ); + if *heap_size < 50 { + assert!( + matches!( + edge, + Err(RootPruneReason::HeapSmallerThanTopK { k: 50, .. }) + ), + "{edge:?}" + ); + small += 1; + } else { + assert!(edge.is_ok(), "{edge:?}"); + } + } + assert_eq!(small, 2 * 28); + } +} diff --git a/crates/accuracy/src/metric_rule.rs b/crates/accuracy/src/metric_rule.rs new file mode 100644 index 000000000..8e4a09996 --- /dev/null +++ b/crates/accuracy/src/metric_rule.rs @@ -0,0 +1,294 @@ +//! The metric rule (concepts doc, section 4, "Metric rule"): the metrics +//! each group's output may be off in, and those each candidate accepts on +//! its inputs. + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId}; + +use crate::metrics_carried; +use crate::readout_caps::caps; + +/// Per group, by group id, the metrics its output may be off in for the +/// queries with an accuracy target that reach it, in +/// `AccuracyMetric::ALL` order: a query root accepts its target's metrics +/// (`AccuracyTarget::metrics`); a candidate's input accepts the metrics its +/// output accepts that its error contract carries ([`metrics_carried`]); a +/// group accepts the union over its consumers. These are the metrics of +/// the readout caps' walk (`readout_caps`), which also reaches a group +/// through inputs the output does not depend on. `None` for a group no +/// such query reaches: no metric rule applies there (accuracy not +/// enforced). `order` is the memo's topological order +/// (`Memo::topological_order`). +pub fn accepted_metrics(memo: &Memo, order: &[MemoGroupId]) -> Vec>> { + caps(memo, order) + .into_iter() + .map(|caps| { + caps.map(|caps| { + AccuracyMetric::ALL + .into_iter() + .filter(|metric| caps.contains_key(metric)) + .collect() + }) + }) + .collect() +} + +/// The metrics `consumer` accepts an input's error in, the metric rule on +/// its input edges, from `accepted` ([`accepted_metrics`]): a readout +/// (`SummaryEstimate`) introduces its error, so it accepts those its own +/// output accepts; any other candidate those its output accepts that its +/// error contract carries ([`metrics_carried`]). `None` where no query +/// with a target reaches the consumer. +pub fn metrics_accepted_on_inputs( + memo: &Memo, + accepted: &[Option>], + consumer: CandidateForMemoGroupId, +) -> Option> { + let accepted = accepted[consumer.group.0 as usize].as_ref()?; + let operator = &memo.candidate(consumer).expression.operator; + Some(match operator { + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) => accepted.clone(), + _ => metrics_carried(memo, operator, accepted), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + producer_satisfies_logical_requirements, readout_metric_of_candidate, NoMeasuredGuarantees, + }; + use asap_types::accuracy::AccuracyTarget; + use asap_types::ir::operator::arguments::Reduction; + use asap_types::ir::operator::AggIntent; + use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; + use asap_types::ir::{ExprSemantics, NonASAPOp, Predicate}; + use asap_types::stages::RootPruneReason; + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; + use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, quantile_over_time, quantile_over_time_readouts, query, + scan_group, QuantileReadouts, + }; + use asap_types::test_support::memo_fixtures::{arithmetic, compare, range}; + use asap_types::workload::DataWorkload; + + fn accepted(memo: &Memo) -> Vec>> { + accepted_metrics(memo, &memo.topological_order().expect("acyclic")) + } + + /// $\alpha^{\log}$ of the edge from `producer` into input `input` of + /// `consumer`, with the metric rule of `memo`'s queries. + fn edge( + memo: &Memo, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Result<(), RootPruneReason> { + producer_satisfies_logical_requirements( + memo, + &readout_metric_of_candidate(memo), + metrics_accepted_on_inputs(memo, &accepted(memo), consumer).as_deref(), + producer, + consumer, + input, + &NoMeasuredGuarantees, + ) + } + + /// The edge on which the readout candidate `id` of a quantile group + /// ([`QuantileReadouts`]) introduces its error into `consumer`'s input + /// `input`: from `id` itself when it is the readout, else from the + /// readout group the renaming `Project` `id` reads. + fn readout_edge( + memo: &Memo, + id: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Result<(), RootPruneReason> { + match &memo.candidate(id).expression.operator { + Operator::NonASAP(NonASAPOp::Project { child, .. }) => { + edge(memo, own_candidate(*child), id, 0) + } + _ => edge(memo, id, consumer, input), + } + } + + /// `quantile_over_time(q, m[5m])` per series of `m` by `job` with KLL + /// and DDSketch readouts beside the exact quantile. + fn quantile_readouts(memo: &mut Memo, q: f64) -> QuantileReadouts { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = group(memo, range(300_000, input)); + quantile_over_time_readouts( + memo, + window, + q, + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + ) + } + + /// A ratio reads no rank error: with a relative target, the KLL + /// readout's metric, rank, is accepted neither by the ratio nor at its + /// own output, so its edge into the division fails the metric rule. + #[test] + fn a_ratio_reads_no_rank_error() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.5 }); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let relative = Some(vec![AccuracyMetric::Relative]); + assert_eq!(accepted(&memo)[p99.0 as usize], relative); + assert_eq!( + metrics_accepted_on_inputs(&memo, &accepted(&memo), own_candidate(ratio)), + relative + ); + assert!(matches!( + edge(&memo, own_candidate(p99), own_candidate(ratio), 0), + Err(RootPruneReason::MetricRule { + producer_metric: AccuracyMetric::Rank, + .. + }) + )); + } + + /// The metric rule cuts edges, not candidates. + /// `avg by (job)(q0.5 / q0.5 + q0.9)` at absolute error 0.05, each + /// quantile per series with KLL, DDSketch and exact candidates: the `+` + /// carries absolute error only, so the division's inputs accept no + /// metric and the edges from the DDSketch and KLL readouts into it are + /// cut; the exact quantile's edge into the division and the division's + /// edge into the `+` hold. + #[test] + fn the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { + let mut memo = Memo::new(&DataWorkload::default()); + let p50 = quantile_readouts(&mut memo, 0.5); + let window = *memo.groups()[p50.result.0 as usize].expression.children()[0]; + let p90 = quantile_over_time_readouts( + &mut memo, + window, + 0.9, + &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], + ); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p50.result, p50.result), + ); + let sum = group( + &mut memo, + arithmetic(ArithmeticOpKind::Add, ratio, p90.result), + ); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: sum, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::ABSOLUTE); + query(&mut memo, avg, Some(target)); + let division = own_candidate(ratio); + assert_eq!( + metrics_accepted_on_inputs(&memo, &accepted(&memo), division), + Some(vec![]) + ); + let result = edge(&memo, division, own_candidate(sum), 0); + assert!(result.is_ok(), "{result:?}"); + for input in 0..2 { + let result = edge(&memo, own_candidate(p50.result), division, input); + assert!(result.is_ok(), "{result:?}"); + for &(algorithm, _, readout) in &p50.by_algorithm { + assert!( + matches!( + readout_edge(&memo, readout, division, input), + Err(RootPruneReason::MetricRule { .. }) + ), + "{algorithm:?}" + ); + } + } + } + + /// A `Filter` whose predicate reads only label columns keeps or drops + /// whole rows, so a value's error passes through it unchanged. + /// `quantile_over_time(0.99, latency[5m])` behind `job = "api"`, + /// relative error 0.05: the filter accepts relative error on its input + /// and the DDSketch readout's edge holds. + #[test] + fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "latency", &["job"], vec![]); + let window = group(&mut memo, range(300_000, input)); + let p99 = + quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); + let job = memo.groups()[p99.result.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the result keeps the job label"); + let filtered = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + job, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )), + child: p99.result, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, filtered, Some(target)); + assert_eq!( + metrics_accepted_on_inputs(&memo, &accepted(&memo), own_candidate(filtered)), + Some(vec![AccuracyMetric::Relative]) + ); + let ddsketch_readout = p99.by_algorithm[0].2; + let result = readout_edge(&memo, ddsketch_readout, own_candidate(filtered), 0); + assert!(result.is_ok(), "{result:?}"); + } + + /// `avg` has an absolute contract only here; a relative bound on a + /// mean needs the values' signs. So + /// `avg(quantile_over_time(0.99, m[5m]))` with a relative target cuts + /// the edges from the sketch readouts into the `avg`: it reads the + /// exact quantile only. + #[test] + fn avg_of_readouts_under_a_relative_target_reads_exact_only() { + let mut memo = Memo::new(&DataWorkload::default()); + let p99 = quantile_readouts(&mut memo, 0.99); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, avg, Some(target)); + for &(algorithm, _, readout) in &p99.by_algorithm { + assert!( + matches!( + readout_edge(&memo, readout, own_candidate(avg), 0), + Err(RootPruneReason::MetricRule { .. }) + ), + "a sketch readout is admissible into avg: {algorithm:?}" + ); + } + let result = edge(&memo, own_candidate(p99.result), own_candidate(avg), 0); + assert!(result.is_ok(), "{result:?}"); + } +} diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs deleted file mode 100644 index 6ed6ad9e7..000000000 --- a/crates/accuracy/src/propagation.rs +++ /dev/null @@ -1,1615 +0,0 @@ -//! Guarantee propagation: the theoretical guarantee of every node of a -//! composed DAG, composed with the built-in analytical rules -//! (`crate::composition::propagate`) through operators, and the helpers -//! Pass 2 and plan selection (Stage 3) use to read, bind and recompose -//! those guarantees and to resize the summaries they depend on. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::accuracy::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, -}; -use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, - SchemaDerivationError, -}; -use asap_types::summary_type::{ - raw_data_samples_of_metric, ExactKind, MergedError, NonNegativeWeightProof, SketchAlgorithm, - SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, -}; - -use crate::composition::{ - additive_as_absolute, propagate, raw_data_samples_sign_statistic, StreamNorm, GROUP_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, -}; - -use crate::function_rules::approximate_input_composition; -use crate::PropagationStats; - -/// A rebuilt node per original node, keyed by the original's address. -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// The approximate inputs a query's value composes: every sketch summary -/// state reachable from `root`, each once, when the value carries a -/// guarantee that is not exact ([`root_guarantee`]), including compositions -/// conditional on source-sign facts supplied later. Empty when neither the -/// current nor conditional contract composes the inputs, or the result is -/// exact. Pass 1 sizes each for its share of failure probability too. -pub fn approximate_inputs(root: &QueryRoot) -> Vec> { - // Sizing may depend on a source-domain proof supplied only at Stage 3. - // Count the inputs of that conditional composition too, without attaching - // the hypothetical proof to the candidate or accepting its accuracy. - let guarantee = root_guarantee(root).or_else(|| { - bind_guarantees(root, &|guarantee| { - guarantee.bind(&|statistic| { - crate::composition::parse_raw_data_samples_sign_statistic(statistic).map(|_| 1.0) - }) - }) - }); - if guarantee.is_none_or(|g| g.is_exact()) { - return Vec::new(); - } - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - let mut seen = HashSet::new(); - operators - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - summary_type: FieldDataType::Sketch(..), - .. - } - ) - ) - }) - .filter(|node| seen.insert(Rc::as_ptr(node))) - .collect() -} - -/// The theoretical guarantee of a query's output, as composed: the root -/// node's own, or a scalar root's composed over the operators it reads -/// (`scalar_guarantee`). -pub fn root_guarantee(root: &QueryRoot) -> Option { - match root { - QueryRoot::Operator(node) => node.guarantee.clone(), - QueryRoot::Scalar(expr) => scalar_guarantee(expr, &|_| None), - } -} - -/// Bind known statistics in existing contracts, then recover compositions -/// that previously required constant child bounds. Physical state contracts, -/// schemas and sharing are preserved. This does not -/// infer facts: `bind` must leave unsupported statistics unknown. -pub fn bind_guarantees( - root: &QueryRoot, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, -) -> Option { - fn walk( - node: &Rc, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, - cache: &mut NodeToRewrittenNodeCache, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return done.clone(); - } - let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); - rebuilt.guarantee = node - .guarantee - .as_ref() - .map(bind) - .or_else(|| composed_guarantee(&rebuilt).as_ref().map(bind)); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - rebuilt - } - let mut cache = HashMap::new(); - let bound = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, bind, &mut cache)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| walk(node, bind, &mut cache))) - } - }; - root_guarantee(&bound) -} - -/// The one source a summary state counts the distinct items of its own -/// rows of, as `source_distinct_guarantee` reads it: the source whose -/// distinct bound gives its failure probability. `None` for a state over -/// computed rows or several sources. -pub fn distinct_bound_source(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }) = - state.asap() - else { - return None; - }; - source_scan(child)?; - source(child) -} - -/// `roots` with every node's guarantee composed again -/// ([`composed_guarantee`]), children first; a sub-DAG shared between roots -/// stays shared. Pass 2 calls it after re-sizing a shared summary, so each -/// consumer's guarantee is the shared state's. -pub fn recompose_guarantees(roots: &[QueryRoot]) -> Result, SchemaDerivationError> { - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_guarantees(node, &mut cache)?) - } - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - with_guarantees(node, &mut cache)?; - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - -/// `node` and everything below it, each node with its [`composed_guarantee`]. -pub fn with_guarantees( - node: &Rc, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - with_guarantees(child, cache)?; - } - let mut rebuilt = node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = composed_guarantee(&rebuilt); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// An extremum over at most one row per group is an identity on its value: -/// an aggregate whose measures are all `MIN`/`MAX` over a (possibly -/// filtered) grouped estimate, where every column of a unique key of the -/// estimate is a group key or pinned to one value by the filter -/// (Pass 2's finer-aggregate same-level read). Unlike a general -/// aggregate over rank estimates, this does not change the population to -/// which the input's error bound refers. -fn selected_group_guarantee(node: &OperatorNode) -> Option { - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - filters, - having: None, - child, - .. - } = node.non_asap()? - else { - return None; - }; - if by.is_without() - || !filters.iter().all(Option::is_none) - || measures.is_empty() - || !measures.iter().all(|measure| { - matches!( - measure, - AggIntent::Min { col: Some(_) } | AggIntent::Max { col: Some(_) } - ) - }) - { - return None; - } - let (pred, grouped) = match child.non_asap()? { - NonASAPOp::Filter { pred, child } => (Some(&pred.0), child), - _ => (None, child), - }; - let pinned = pred.map_or_else(Vec::new, pinned_columns); - // The filter may read only key columns: a predicate on an estimate - // would decide by an approximate value which groups are kept. - let reads = pred.map_or_else(Vec::new, ScalarExpr::columns_referenced); - if !grouped.schema.unique_keys.iter().any(|key| { - key.iter() - .all(|column| by.keys().contains(column) || pinned.contains(column)) - && reads.iter().all(|column| key.contains(column)) - }) { - return None; - } - let mut source = grouped; - while let Some(NonASAPOp::Project { cols, child, .. }) = source.non_asap() { - if !cols - .iter() - .enumerate() - .all(|(index, item)| matches!(item.expr, ScalarExpr::Column(column) if column == index)) - { - return None; - } - source = child; - } - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = source.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(keys), - child: raw, - .. - }) = summary_input.asap() - else { - return None; - }; - if keys.is_without() - || !raw - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return None; - } - grouped.guarantee.clone() -} - -/// The theoretical guarantee of a composed node, given its children's, -/// composed with the built-in analytical rules (`propagate`); -/// Stage 1 reads no data statistics or deployment evidence. Exact over exact -/// inputs; a summary estimate has its sized family's bound when -/// `estimate_has_bound`, composed with its input's when that is -/// approximate; an exact aggregation or accumulator over approximate values -/// uses its function's rule (`approximate_input_composition`); arithmetic composes as -/// `arithmetic_guarantee`. A rule that needs a statistic leaves it -/// symbolic (`BoundExpr::Unknown`), and plan selection (Stage 3) binds it. -/// `None` where no rule applies: a comparison, a value `HAVING` or any other -/// predicate over an approximate value, a rank error through any operator (a -/// rank interval counts as none in the MVP). A projection of columns only -/// keeps its child's guarantee. A summary state carries none itself. -/// -/// A guarantee holds per output point (per series or group), with its -/// failure probability per point ([`AccuracyScope::PerValue`]). Plan -/// selection states it over a whole result of `G` points as `G·δ`. -/// -/// [`AccuracyScope::PerValue`]: asap_types::accuracy::AccuracyScope::PerValue -pub fn composed_guarantee(node: &OperatorNode) -> Option { - if let Some(guarantee) = selected_group_guarantee(node) { - return Some(guarantee); - } - if let Some(guarantee) = rolled_up_guarantee(node) { - return guarantee; - } - if let Some(guarantee) = merged_tumbling_windows_guarantee(node) { - return guarantee; - } - let exact = |node: &OperatorNode| { - node.guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - }; - // A summary state's input rows and any subquery in its filter. - let exact_inputs = |state: &OperatorNode| state.children().into_iter().all(|c| exact(c)); - // Over approximate input rows, a summary's filter must not decide which - // rows count by an estimate: it may read no input column (guarantees are - // per node, so any column may carry the estimate; column-level tracking - // is #689), and its subqueries must be exact. - let exact_filter = |state: &OperatorNode| { - let Some(ASAPOp::SummaryAgg { filter, .. }) = state.asap() else { - return false; - }; - filter - .as_ref() - .is_none_or(|filter| !reads_columns(&filter.0)) - && state.children().into_iter().skip(1).all(|c| exact(c)) - }; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(..), - .. - }) if exact_inputs(child) => Some(AccuracyGuarantee::exact( - "exact accumulator over exact input", - )), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(kind, _), - child: input, - reduction, - .. - }) if exact_filter(child) && !groups_by_estimate(reduction, input) => { - let stats = fold_stats(reduction, input)?; - let input = operand_guarantee(input)?; - let op = match kind { - // Counts rows, not values. - ExactKind::Count => return Some(AccuracyGuarantee::exact("count of rows")), - ExactKind::Sum => CompositionOperator::ExactSum, - ExactKind::Min | ExactKind::Max => CompositionOperator::ExactExtremum, - ExactKind::Rate => CompositionOperator::CounterRate, - ExactKind::IRate => CompositionOperator::InstantCounterRate, - ExactKind::Increase => CompositionOperator::CounterIncrease, - }; - propagate(&op, &[input], None, &stats).ok() - } - _ => None, - }, - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => match summary_input.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: family, - input: update, - child: input, - reduction, - .. - }) if estimate_has_bound(family, update, query) - || estimate_bound_over_raw_data_samples(family, update, query).is_some() => - { - let local = match estimate_bound_over_raw_data_samples(family, update, query) { - Some(gated) => gated, - None => source_distinct_guarantee(family, query, input) - .or_else(|| { - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters") - })?, - }; - let local = interpolation_guarantee(family, update, query, input, local)?; - if exact_inputs(summary_input) { - Some(local) - } else if exact_filter(summary_input) && !groups_by_estimate(reduction, input) { - compose( - &CompositionOperator::ApproximateAggregate, - vec![input.guarantee.clone()?], - Some(&local), - ) - } else { - None - } - } - _ => None, - }, - Operator::ASAP(_) => None, - // Renaming or reordering columns keeps the child's guarantee. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - child.guarantee.clone() - } - // Each step re-evaluates the child: every sample keeps its guarantee. - Operator::NonASAP(NonASAPOp::PromqlSubquery { child, .. }) => child.guarantee.clone(), - Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { - Some(AccuracyGuarantee::exact("exact over exact inputs")) - } - // Over a join, the child's sample value may be an exact input's - // column, so each column's own guarantee is read instead. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) if derives_from_join(child) => { - relational_project_guarantee(cols, child) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), - Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), - // A measure filter or `HAVING` over approximate values changes - // which rows or groups count: no rule. - Operator::NonASAP(NonASAPOp::Aggregate { - measures, - filters, - having: None, - reduction, - child, - .. - }) if filters.iter().all(Option::is_none) && !groups_by_estimate(reduction, child) => { - match measures.as_slice() { - [intent] => propagate( - &approximate_input_composition(intent)?, - &[operand_guarantee(child)?], - None, - &fold_stats(reduction, child)?, - ) - .ok(), - _ => None, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => arithmetic_guarantee( - op.clone(), - (operand_guarantee(lhs), None), - (operand_guarantee(rhs), None), - Rc::ptr_eq(lhs, rhs), - ), - Operator::NonASAP(_) => None, - } -} - -/// The guarantee of an estimate or finalized value read from a rollup (a -/// regrouping `SummaryMerge` of one input): the same read of the input -/// states, when the family's merged error for that read is exact or the -/// same as one build's over the union (the family-capability table). The -/// groups of one build partition its rows, so a rolled-up state is a merge -/// of disjoint parts, and a family whose bound holds for one build over -/// the union keeps it: KMV, HLL and DDSketch merge into the state a direct -/// coarse build would have, KLL into a state with the same rank bound, and -/// linear sketches into the direct build's counters, whose error is -/// relative to the coarse group's total. Each guarantee holds per output -/// point, so the number of groups does not change it. None when the merged -/// readout is approximate (heap top-k, entropy) or the family has no merge. -/// `None` for any node that does not read a rollup. -fn rolled_up_guarantee(node: &OperatorNode) -> Option> { - let state = match node.asap()? { - ASAPOp::SummaryEstimate { summary_input, .. } => summary_input, - ASAPOp::FinalizeExactAccumulator { child } => child, - _ => return None, - }; - let Some(ASAPOp::SummaryMerge { - children, - group_by: Some(_), - .. - }) = state.asap() - else { - return None; - }; - let [input] = children.as_slice() else { - return Some(None); - }; - // Kept whole, a selected state keeps the guarantee it was built with. - let input = - asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); - let family = state.asap()?.produced_state()?; - let merged = match node.asap()? { - ASAPOp::SummaryEstimate { query, .. } => { - SummaryTypeCapability::of(family).merged_error(query) - } - // An exact accumulator merged across groups is exact. - _ => SummaryTypeCapability::merges_groups(family).then_some(MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - let consumer = OperatorNode::with_schema( - node.operator.map_children(|child| { - if Rc::ptr_eq(child, state) { - Rc::clone(&input) - } else { - Rc::clone(child) - } - }), - node.schema.clone(), - ); - Some(composed_guarantee(&consumer)) -} - -/// The guarantee of an estimate or finalized value read from merged -/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a -/// chain): the guarantee of the same read of one build over the time range -/// the merged windows cover when the -/// summary type's merged error for that read is exact or the same as that build's -/// (the summary-type capability table); none when it is approximate (evidence -/// only, for plan selection to judge) or the family has no merge. `None` for -/// any node that does not read merged tumbling windows. -fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, query) = match node.asap()? { - ASAPOp::SummaryEstimate { - summary_input, - query, - } => (summary_input, Some(query)), - ASAPOp::FinalizeExactAccumulator { child } => (child, None), - _ => return None, - }; - let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { - return None; - }; - let capability = SummaryTypeCapability::of(&summary_type); - let merged = match query { - Some(query) => capability.merged_error(query), - // An exact accumulator's merged state is exact. - None => capability.merge_inputs.map(|_| MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - todo!( - "#1207 PR 4: the guarantee of a read of merged tumbling windows, as of one build over their window, over memo groups" - ) -} - -/// The guarantee of a distinct count of one instance per group over a -/// source's own rows, whose failure probability is left symbolic in the -/// source's distinct bound, which Stage 1 does not read: -/// - classic HLL's ([`hll::classic_guarantee`]); -/// - UnivMon's, sized for a distinct count ([`univmon::distinct_guarantee`]). -/// -/// `None` for any other summary, or one over computed rows, whose distinct -/// items the source's bound does not bound. -pub(crate) fn source_distinct_guarantee( - family: &FieldDataType, - query: &SketchStatistic, - input: &Rc, -) -> Option { - let FieldDataType::Sketch(algorithm) = family else { - return None; - }; - if *query != SketchStatistic::Cardinality { - return None; - } - source_scan(input)?; - match algorithm { - SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { - todo!( - "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" - ) - } - _ => None, - } -} - -/// The rows one group of an exact fold over approximate `input` counts. -/// Across PromQL series (`Reduce`) they are at most the input series, an -/// input-row-count statistic left symbolic for plan selection; across SQL -/// groups they are a group-row-count statistic nothing binds; over time -/// (`PerEntity`) they are one series' samples in the window: the steps of a -/// subquery with an explicit resolution, `⌈range / resolution⌉`. `None` -/// when that count is not in the query. -pub(crate) fn fold_stats( - reduction: &Reduction, - input: &Rc, -) -> Option { - let sign_statistic = - estimates_raw_data_samples_of(input).map(|metric| raw_data_samples_sign_statistic(&metric)); - let stats = match reduction { - // The series count of the sources read bounds a PromQL fold's rows - // (series), not the rows of a SQL group. - Reduction::Reduce(_) if reads_table(input) => Some(PropagationStats { - row_count_statistic: Some(GROUP_ROW_COUNT), - ..PropagationStats::default() - }), - Reduction::Reduce(_) => Some(PropagationStats::default()), - Reduction::PerEntity => match input.non_asap() { - Some(NonASAPOp::PromqlSubquery { - range, - resolution: Some(resolution), - .. - }) if !resolution.is_zero() => Some(PropagationStats { - input_row_count: Some(range.as_nanos().div_ceil(resolution.as_nanos()) as u64), - ..PropagationStats::default() - }), - _ => None, - }, - }?; - Some(PropagationStats { - sign_statistic, - ..stats - }) -} - -/// The metric whose raw data samples, or plain sums of them, a composed node's -/// values are or estimate, or `None`: [`raw_data_samples_of_metric`]'s strict rule -/// over a composed candidate. An exact sum accumulator, a projection of -/// columns and a -/// heap estimate whose update weights carry the -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement keep it, and so does a -/// quantile estimate of those values, built over the raw window or merged -/// from its tumbling windows: it is one of them, or between two of them, so it -/// shares their sign. Anything else (rate, arithmetic, other -/// functions) ends it. Only the sign of the values is read from it. -pub(crate) fn estimates_raw_data_samples_of(node: &Rc) -> Option { - match &node.operator { - Operator::NonASAP(NonASAPOp::TimeRange { child, .. }) => { - estimates_raw_data_samples_of(child) - } - Operator::NonASAP(NonASAPOp::TimeShift { child, shift }) if shift.at.is_none() => { - estimates_raw_data_samples_of(child) - } - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap()? { - ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), - child, - filter: None, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, - // Only a φ in [0, 1] picks one of the values; outside it the answer - // is -Inf or +Inf, whatever their sign. Merged tumbling windows of one - // definition are read through their `SummaryTumblingWindowChain`: - // a quantile of the merged state is still one of the tumbling windows' - // samples, or between two of them. - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::Quantile { q }, - }) if (0.0..=1.0).contains(q) => { - let build = match summary_input.summary_build()? { - chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, - _ => summary_input.asap()?.clone(), - }; - match &build { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - } - } - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - match summary_input.asap()? { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - weight_domain: WeightDomain::RawDataSamplesOfMetric { metric }, - .. - }, - filter: None, - .. - } => Some(metric.clone()), - _ => None, - } - } - // Renaming, reordering or dropping columns keeps every value. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - estimates_raw_data_samples_of(child) - } - _ => raw_data_samples_of_metric(node), - } -} - -/// Whether `node` reads a SQL table. -fn reads_table(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Scan { - source: Source::Table { .. }, - .. - }) - ) - }) -} - -/// Whether a grouping by `reduction` over an approximate `input` may decide -/// a row's group by an estimate: a grouping column other than a label (a -/// string) or the time index. Guarantees are per node, so any such column -/// may carry the estimate (column-level tracking is #689); a PromQL `by` -/// or `without` groups by labels only. `false` over exact input. -fn groups_by_estimate(reduction: &Reduction, input: &OperatorNode) -> bool { - let Some(by) = reduction.group_keys() else { - return false; - }; - if input - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return false; - } - let schema = &input.schema; - let columns: Vec = if by.is_without() { - let value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); - (0..schema.fields.len()) - .filter(|i| !by.keys().contains(i) && Some(*i) != value) - .collect() - } else { - by.keys().to_vec() - }; - columns.into_iter().any(|i| { - Some(i) != schema.time_index - && !matches!( - schema.fields.get(i).map(|f| &f.dtype), - Some(FieldDataType::Plain(DataType::Utf8)) - ) - }) -} - -/// Whether `expr` reads any input column. -fn reads_columns(expr: &ScalarExpr) -> bool { - let mut reads = false; - expr.map_columns(&mut |column| { - reads = true; - column - }); - reads -} - -/// The built-in rule for `op`, with every statistic unknown (Stage 1 reads -/// none); `None` when no rule applies. -fn compose( - op: &CompositionOperator, - inputs: Vec, - local: Option<&AccuracyGuarantee>, -) -> Option { - propagate(op, &inputs, local, &PropagationStats::default()).ok() -} - -/// A projection that computes over its child's sample value: the guarantee -/// of the one item that reads it ([`project_column_guarantee`]). `None` when -/// no single item reads the value. -fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option { - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - let reads_value = |expr: &ScalarExpr| { - let mut reads = false; - expr.map_columns(&mut |column| { - reads |= column == value; - column - }); - reads - }; - let [item] = <[_; 1]>::try_from( - cols.iter() - .filter(|item| reads_value(&item.expr)) - .collect::>(), - ) - .ok()?; - scalar_guarantee(&item.expr, &|column| { - project_column_guarantee(child, column) - }) -} - -/// Whether `node` is a join, or reads one only through operators that keep -/// its columns (projections, filters, sorts, limits, deduplication). The -/// columns of such a node come from different inputs, so its sample value -/// says nothing about where the error lies. -fn derives_from_join(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Join { .. }) => true, - Some( - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } - | NonASAPOp::Dedup { child, .. }, - ) => derives_from_join(child), - _ => false, - } -} - -/// The guarantee of each output column of `node`, where it is known: every -/// column of an exact node is exact; a join beside exact inputs -/// ([`join_guarantee`]) gives the exact inputs' columns exact and the -/// approximate input's columns its guarantee; a projection of columns maps -/// its child's. `None` anywhere else, a computed projection included. -/// Guarantees are per node, so every column of an approximate input counts -/// as approximate; column-level tracking is #689. -/// -/// This assumes every approximate node of Stage 1 has an exact row set: -/// its guarantee bounds each row's values, never which rows exist (a -/// top-k estimate, whose rows are approximate, has no bound; -/// `estimate_has_bound`). A guarantee on row membership would need these -/// rules revisited. -fn column_guarantees(node: &OperatorNode) -> Option> { - let width = node.schema.fields.len(); - if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { - return Some(vec![guarantee.clone(); width]); - } - match &node.operator { - Operator::NonASAP(NonASAPOp::Join { - kind: - JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - left, - right, - .. - }) => { - let input = join_input_guarantees(left, right)?; - (input.len() == width).then_some(input) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { - let child = column_guarantees(child)?; - cols.iter() - .map(|item| match item.expr { - ScalarExpr::Column(id) => child.get(id).cloned(), - _ => None, - }) - .collect() - } - _ => None, - } -} - -/// The guarantees of a join's input columns (`left ++ right`) when at most -/// one input is approximate and the others are exact: the exact input's -/// columns exact, the approximate input's columns its guarantee. -fn join_input_guarantees( - left: &OperatorNode, - right: &OperatorNode, -) -> Option> { - let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); - if !exact(left) && !exact(right) { - return None; - } - let side = |node: &OperatorNode| { - let guarantee = node.guarantee.clone()?; - Some(vec![guarantee; node.schema.fields.len()]) - }; - let mut columns = side(left)?; - columns.extend(side(right)?); - Some(columns) -} - -/// A join whose inputs are exact but one keeps that input's guarantee when -/// its predicate reads only exact columns (a cross join, or a key equality -/// on the exact inputs' columns): which rows match, and how often, then -/// does not depend on an estimate, and each output row carries one row of -/// the approximate input unchanged. Outer joins pad on the same exact -/// decision. A semi- or anti-join, a key on an approximate input's column -/// (per-node guarantees count all its columns approximate; #689), or two -/// approximate inputs have none. An exact join is composed before this. -/// Like [`column_guarantees`], this assumes the approximate input's row set -/// is exact: only its values carry error. -fn join_guarantee(node: &OperatorNode) -> Option { - let Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - pred, - left, - right, - }) = &node.operator - else { - return None; - }; - let columns = join_input_guarantees(left, right)?; - let pred_exact = - scalar_guarantee(&pred.0, &|id| columns.get(id).cloned()).is_some_and(|g| g.is_exact()); - if !pred_exact { - return None; - } - [left, right] - .into_iter() - .find_map(|input| input.guarantee.clone().filter(|g| !g.is_exact())) -} - -/// The guarantee of a projection over relational columns -/// ([`column_guarantees`]): each output column's as [`scalar_guarantee`] -/// (a `CASE` as [`case_guarantee`]), and the node's the weakest of them. -/// `None` when the child's column guarantees are unknown or any output -/// column has none. -fn relational_project_guarantee( - cols: &[ProjectItem], - child: &Rc, -) -> Option { - let columns = column_guarantees(child)?; - let column = |id: usize| columns.get(id).cloned(); - weakest( - cols.iter() - .map(|item| scalar_guarantee(&item.expr, &column)) - .collect::>>()?, - ) -} - -/// The weakest of `guarantees`: exact when all are, the approximate one -/// when there is one (or all approximate ones are equal), and otherwise -/// as an extremum of them ([`lipschitz_guarantee`]). -fn weakest(guarantees: Vec) -> Option { - let mut approximate: Vec = Vec::new(); - for guarantee in guarantees.into_iter().filter(|g| !g.is_exact()) { - if !approximate.contains(&guarantee) { - approximate.push(guarantee); - } - } - match approximate.len() { - 0 => Some(AccuracyGuarantee::exact("exact over exact operands")), - 1 => approximate.pop(), - _ => lipschitz_guarantee(approximate), - } -} - -/// A `CASE` whose operand and conditions are exact picks its branch by an -/// exact decision, so it keeps the weakest branch's guarantee (a literal or -/// NULL branch, or a missing `ELSE`, is exact). A condition over an -/// approximate value has none. -fn case_guarantee( - operand: Option<&ScalarExpr>, - branches: &[(ScalarExpr, ScalarExpr)], - else_expr: Option<&ScalarExpr>, - column: &dyn Fn(usize) -> Option, -) -> Option { - let exact = |expr: &ScalarExpr| scalar_guarantee(expr, column).is_some_and(|g| g.is_exact()); - if !operand.is_none_or(exact) || !branches.iter().all(|(when, _)| exact(when)) { - return None; - } - weakest( - branches - .iter() - .map(|(_, then)| then) - .chain(else_expr) - .map(|expr| scalar_guarantee(expr, column)) - .collect::>>()?, - ) -} - -/// The guarantee of `column` of a projection's `child`: over a join, the -/// column's own (`column_guarantees`); otherwise the child's sample -/// value has the child's as an operand ([`operand_guarantee`]) and other -/// columns (labels, timestamps) are exact. `None` when the child has no -/// sample value or the column's guarantee is unknown. -pub fn project_column_guarantee( - child: &Rc, - column: usize, -) -> Option { - // Over a join, each column's own guarantee ([`column_guarantees`]). - if derives_from_join(child) { - return column_guarantees(child)?.get(column).cloned(); - } - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - if column == value { - operand_guarantee(child) - } else { - Some(AccuracyGuarantee::exact( - "column other than the sample value", - )) - } -} - -/// The guarantee of a scalar expression, given each column's: `scalar(v)` -/// and a SQL scalar subquery have their relation's, a literal or the -/// evaluation time is exact, arithmetic composes as -/// [`arithmetic_guarantee`], and `clamp`/`clamp_min`/`clamp_max`/`abs` as -/// [`lipschitz_guarantee`]. Any other expression (a comparison, a rounding, -/// a non-linear function) is exact over exact operands and has none -/// otherwise. -pub(crate) fn scalar_guarantee( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - let literal = |expr: &ScalarExpr| match expr { - ScalarExpr::Literal(ScalarValue::Float64(value)) => Some(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Some(*value as f64), - _ => None, - }; - let operand = |expr: &ScalarExpr| (operand_of(expr, column), literal(expr)); - match expr { - ScalarExpr::Column(id) => column(*id), - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - node.guarantee.clone() - } - ScalarExpr::Literal(_) | ScalarExpr::CurrentTimestamp | ScalarExpr::EvalTimestamp => { - Some(AccuracyGuarantee::exact("constant")) - } - ScalarExpr::Negative { expr, .. } => { - lipschitz_guarantee(vec![scalar_guarantee(expr, column)?]) - } - ScalarExpr::Case { - operand, - branches, - else_expr, - } => case_guarantee(operand.as_deref(), branches, else_expr.as_deref(), column), - ScalarExpr::Arithmetic { - op, left, right, .. - } => arithmetic_guarantee( - op.clone(), - operand(left), - operand(right), - same_value(left, right), - ), - ScalarExpr::FunctionCall { name, args } if is_monotone_lipschitz(name) => { - lipschitz_guarantee( - args.iter() - .map(|arg| operand_of(arg, column)) - .collect::>()?, - ) - } - _ => { - let mut columns = Vec::new(); - expr.map_columns(&mut |id| { - columns.push(id); - id - }); - let exact = |g: Option| g.is_some_and(|g| g.is_exact()); - (columns.into_iter().all(|id| exact(column(id))) - && expr - .operator_refs() - .into_iter() - .all(|node| exact(node.guarantee.clone()))) - .then(|| AccuracyGuarantee::exact("exact over exact operands")) - } - } -} - -/// The guarantee of a scalar operand as an input to arithmetic or a -/// function: a planned operator's as [`operand_guarantee`], anything else -/// as `scalar_guarantee`. -pub fn operand_of( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - operand_guarantee(node) - } - _ => scalar_guarantee(expr, column), - } -} - -/// `node`'s guarantee as an input to a computation over its values: a -/// Count-Min or Count Sketch bound restated as an absolute error over the -/// stream norm its summary's update weight allows (`stream_norm`, -/// `additive_as_absolute`). -pub fn operand_guarantee(node: &Rc) -> Option { - let guarantee = node.guarantee.clone()?; - Some(additive_as_absolute(guarantee, stream_norm(node).as_ref())) -} - -/// The stream norm that bounds the one Count-Min or Count Sketch summary -/// below `node`, by its update weight: the source's own norm when the -/// weight is the source's raw value column (the summary's input is the -/// source's scan, at most windowed or filtered); `|c|` times the source's -/// row count for a constant weight `c`. `None` for any other weight (a -/// computed value such as `m * 1000` or `rate(…)`), several summaries or -/// sources, or none. -pub(crate) fn stream_norm(node: &Rc) -> Option { - let summaries: Vec<_> = OperatorNode::reachable(node) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(algorithm), - .. - }) if matches!( - algorithm, - SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap - ) - ) - }) - .collect(); - let [summary] = summaries.as_slice() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - input: update, - child, - .. - }) = summary.asap() - else { - return None; - }; - let source = source(child)?; - match &update.weight { - SummaryInputExpr::Constant(weight) => Some(StreamNorm::ConstantWeight { - weight: *weight, - source, - }), - SummaryInputExpr::Column(column) if raw_value_column(child, column) => { - Some(StreamNorm::Source(source)) - } - _ => None, - } -} - -/// Whether `column` of `node` is its source's raw data sample value: `node` is -/// the source's scan, at most behind filters and windows that keep its -/// schema, and `column` resolves to the scan's sample value. -fn raw_value_column(node: &Rc, column: &ColumnRef) -> bool { - let Some(schema) = source_scan(node) else { - return false; - }; - let resolved = resolve_column_ref(column, schema).ok(); - resolved.is_some() && resolved == resolve_column_ref(&ColumnRef::SampleValue, schema).ok() -} - -/// The scan's schema when `node` is a source's scan, at most behind filters -/// and windows that keep its schema: its rows are the source's own. -fn source_scan(node: &Rc) -> Option<&Schema> { - let mut current = node; - loop { - if current.schema != node.schema { - return None; - } - match current.non_asap() { - Some(NonASAPOp::Filter { child, .. } | NonASAPOp::TimeRange { child, .. }) => { - current = child - } - Some(NonASAPOp::Scan { schema, .. }) => return Some(schema), - _ => return None, - } - } -} - -/// The one source (PromQL metric or SQL table) `node` reads; `None` when it -/// reads several or none. -fn source(node: &Rc) -> Option { - let sources: HashSet = OperatorNode::reachable(node) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => Some(match source { - Source::TimeSeries { metric } => metric.clone(), - Source::Table { table_ref } => table_ref.clone(), - }), - _ => None, - }) - .collect(); - let mut sources = sources.into_iter(); - match (sources.next(), sources.next()) { - (Some(source), None) => Some(source), - _ => None, - } -} - -/// Whether two scalar operands are one value: the same column, or the same -/// planned operator read twice. -fn same_value(left: &ScalarExpr, right: &ScalarExpr) -> bool { - match (left, right) { - (ScalarExpr::Column(a), ScalarExpr::Column(b)) => a == b, - ( - ScalarExpr::PromqlScalarFromVector(a) | ScalarExpr::ScalarSubquery(a), - ScalarExpr::PromqlScalarFromVector(b) | ScalarExpr::ScalarSubquery(b), - ) => Rc::ptr_eq(a, b), - _ => false, - } -} - -/// Functions that are 1-Lipschitz and, for the relative-error rule, -/// monotone in each argument with `|f(x)|` scaling with `|x|`: `clamp` -/// (`min(max(x, lo), hi)`), `clamp_min`, `clamp_max` and `abs`. -fn is_monotone_lipschitz(name: &str) -> bool { - matches!( - name, - "promql_clamp" | "promql_clamp_min" | "promql_clamp_max" | "promql_abs" | "abs" - ) -} - -/// The guarantee of a 1-Lipschitz function of `args` (negation, `abs`, -/// `clamp`): its value is within the largest of its arguments' absolute -/// errors, by the union bound over them; likewise for relative errors, as -/// for an exact `max`/`min` (`ExactExtremum`, one value per argument). -/// Count-Min and Count Sketch bounds count as absolute errors. `None` for a -/// rank error or a mix of absolute and relative ones. -fn lipschitz_guarantee(args: Vec) -> Option { - let approximate: Vec<_> = args - .into_iter() - .filter(|g| !g.is_exact()) - .map(|g| additive_as_absolute(g, None)) - .collect(); - if approximate.is_empty() { - return Some(AccuracyGuarantee::exact( - "Lipschitz function of exact operands", - )); - } - propagate( - &CompositionOperator::ExactExtremum, - &approximate, - None, - &PropagationStats { - input_row_count: Some(1), - ..PropagationStats::default() - }, - ) - .ok() -} - -/// An arithmetic operand: its guarantee, and its value when it is a literal. -type Operand = (Option, Option); - -/// The guarantee of `left op right` (#669, MVP). Count-Min and Count Sketch -/// bounds count as absolute errors (normalized bound times stream norm). -/// - `x ± y`: absolute errors add (union bound on the failure -/// probabilities); `x ± c` (`c` exact) keeps an absolute error. -/// - `c · x`, `x / c` (`c` a nonzero literal): an absolute error scales by -/// `|c|` (or `1/|c|`); a relative error is kept, as it is by `x · y` or -/// `x / y` with `y` exact. -/// - `x · y` of two relative errors: relative `a + b + a·b`. -/// - `x / y` with `y` a relative error and `x` a relative error or exact: -/// relative `(a + b)/(1 − b)` (`a = 0` for an exact `x`), none when -/// `b ≥ 1`. -/// - `x · y` of two other estimates, `x / y` with any other approximate -/// `y`: a bound that needs enforced operand domains, left symbolic -/// ([`operand_domain_statistic`]) for plan selection to bind. -/// - Everything else (a relative error shifted by a constant or added to -/// another, a rank error, `%`, `^`, `atan2`) has none. -/// -/// `same_operand`: both sides are one estimate (the same planned operator), -/// which fails or holds as one event, so its failure probability counts -/// once. Two estimates count once each, even over one shared summary: the -/// built-in guarantees hold per estimate, not simultaneously. -fn arithmetic_guarantee( - op: ArithmeticOpKind, - left: Operand, - right: Operand, - same_operand: bool, -) -> Option { - let (l, r) = ( - additive_as_absolute(left.0?, None), - additive_as_absolute(right.0?, None), - ); - let once = same_operand.then(|| l.spec.failure_probability.clone()); - let statistic = operand_domain_statistic(&op, &l, &r); - let composed = match (op, l.is_exact(), r.is_exact()) { - (_, true, true) => return Some(AccuracyGuarantee::exact("arithmetic over exact operands")), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, true, false) => shifted(r), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, true) => shifted(l), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, false) => compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![l], - Some(&r), - ), - (ArithmeticOpKind::Mul, true, false) => scaled(r, left.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, true) => scaled(l, right.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, false) if statistic.is_none() => relative_product(l, r), - (ArithmeticOpKind::Div, false, true) => match right.1 { - Some(0.0) => None, - c => scaled(l, c.map(|c| 1.0 / c.abs())), - }, - (ArithmeticOpKind::Div, _, false) if statistic.is_none() => propagate( - &CompositionOperator::ExactDivision, - &[l, r], - None, - &PropagationStats::default(), - ) - .ok(), - (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, _, _) => symbolic(l, r, statistic?), - _ => None, - }?; - let mut composed = composed; - if let Some(failure_probability) = once { - composed.spec.failure_probability = failure_probability; - } - Some(composed) -} - -/// The statistic a product or quotient of `l` and `r` leaves symbolic on -/// its operands' domains, when it is one whose bound needs them: a product -/// of two estimates, other than of two relative errors -/// ([`PRODUCT_OPERAND_DOMAINS`]); a quotient with an approximate -/// denominator, other than of relative or exact over relative -/// (`QUOTIENT_OPERAND_DOMAINS`). `None` for any other operation (a -/// product or quotient of relative errors has a closed form), or when an -/// operand's error has no value form (a rank error). Plan selection -/// (Stage 3) finds the operations to bind with this same rule. -pub fn operand_domain_statistic( - op: &ArithmeticOpKind, - l: &AccuracyGuarantee, - r: &AccuracyGuarantee, -) -> Option<&'static str> { - let valued = |g: &AccuracyGuarantee| { - g.is_exact() - || matches!( - g.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE | AccuracyMetric::Relative - ) - }; - let relative = - |g: &AccuracyGuarantee| g.is_exact() || g.spec.accuracy_metric == AccuracyMetric::Relative; - if !(valued(l) && valued(r)) { - return None; - } - match op { - ArithmeticOpKind::Mul - if !l.is_exact() - && !r.is_exact() - && !(l.spec.accuracy_metric == AccuracyMetric::Relative - && r.spec.accuracy_metric == AccuracyMetric::Relative) => - { - Some(PRODUCT_OPERAND_DOMAINS) - } - ArithmeticOpKind::Div if !r.is_exact() && !(relative(l) && relative(r)) => { - Some(QUOTIENT_OPERAND_DOMAINS) - } - _ => None, - } -} - -/// A product of two relative-error estimates: within relative -/// `(1 + a)(1 + b) − 1 = a + b + a·b`, whatever the signs. `None` for any -/// other pair. -fn relative_product(l: AccuracyGuarantee, r: AccuracyGuarantee) -> Option { - if l.spec.accuracy_metric != AccuracyMetric::Relative - || r.spec.accuracy_metric != AccuracyMetric::Relative - { - return None; - } - let bound = BoundExpr::Sum { - terms: vec![ - l.spec.error.clone(), - r.spec.error.clone(), - BoundExpr::Product { - factors: vec![l.spec.error.clone(), r.spec.error.clone()], - }, - ], - }; - Some(binary_guarantee( - AccuracyMetric::Relative, - bound, - l, - r, - None, - CompositionOperator::ExactProduct, - "relative_product_union_bound", - )) -} - -/// `x` times an exact factor (`Some(|c|)` when it is a literal): a -/// relative error is kept; an absolute error scales by `|c|`. -fn scaled(input: AccuracyGuarantee, factor: Option) -> Option { - match (input.spec.accuracy_metric, factor) { - (AccuracyMetric::Relative, _) => Some(input), - (_, Some(constant)) => compose( - &CompositionOperator::Lipschitz { constant }, - vec![input], - None, - ), - _ => None, - } -} - -/// `x` plus an exact value: an absolute error is kept. -fn shifted(input: AccuracyGuarantee) -> Option { - compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![input], - None, - ) -} - -/// A product or quotient whose bound needs its operands' domains, which -/// Stage 1 does not read: the bound is the symbolic `statistic` -/// ([`operand_domain_statistic`]), absolute, with the union bound on the -/// failure probability. -fn symbolic( - l: AccuracyGuarantee, - r: AccuracyGuarantee, - statistic: &'static str, -) -> Option { - let (operator, rule) = match statistic { - QUOTIENT_OPERAND_DOMAINS => ( - CompositionOperator::ExactDivision, - "absolute_division_needs_operand_domains", - ), - _ => ( - CompositionOperator::ExactProduct, - "absolute_product_needs_operand_domains", - ), - }; - Some(binary_guarantee( - AccuracyMetric::ABSOLUTE, - BoundExpr::Unknown { - statistic: statistic.into(), - }, - l, - r, - Some(statistic), - operator, - rule, - )) -} - -/// A guarantee over two operands with the union bound on their failure -/// probabilities, and its provenance. -fn binary_guarantee( - metric: AccuracyMetric, - bound: BoundExpr, - l: AccuracyGuarantee, - r: AccuracyGuarantee, - unavailable: Option<&str>, - operator: CompositionOperator, - rule: &str, -) -> AccuracyGuarantee { - let failure_probability = ProbabilityExpr::UnionBound { - terms: vec![ - l.spec.failure_probability.clone(), - r.spec.failure_probability.clone(), - ], - }; - let provenance = [l, r] - .into_iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g), - }) - .chain( - unavailable.map(|statistic| GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }), - ) - .chain([GuaranteeSource::CompositionStep { - operator, - rule: rule.into(), - }]) - .collect(); - AccuracyGuarantee::new(metric, bound, failure_probability, provenance) -} - -/// Whether the default accuracy model's bound for `family` answers `query` -/// over `update`. A heap sketch's bound is per item frequency, not Top-K -/// membership; a Count-Min bound assumes non-negative weights. -pub(crate) fn estimate_has_bound( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> bool { - let count_min = matches!( - family, - FieldDataType::Sketch(algorithm) - if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) - ); - !matches!(query, SketchStatistic::TopK { .. }) - && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) -} - -/// DDS quantiles use linear interpolation of neighbouring representatives. -/// Relative error survives that convex combination only without cancellation. -/// Endpoints select an extremum; interior quantiles need a sign proof. Keep a -/// raw metric's requirement symbolic so Stage 3 can bind declared source facts. -fn interpolation_guarantee( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, - input: &Rc, - mut guarantee: AccuracyGuarantee, -) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch)) - && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); - if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { - return Some(guarantee); - } - let metric = match &update.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => metric.clone(), - _ if matches!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ) => - { - estimates_raw_data_samples_of(input)? - } - _ => return None, - }; - guarantee.spec.error = asap_types::accuracy::BoundExpr::Product { - factors: vec![ - guarantee.spec.error, - asap_types::accuracy::BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic(&metric), - }, - ], - }; - Some(guarantee) -} - -/// A Count-Min estimate whose weights carry the open requirement that they -/// are raw data samples of one metric ([`WeightDomain::RawDataSamplesOfMetric`]): its local -/// guarantee with the bound gated on that metric's sign -/// ([`raw_data_samples_sign_statistic`]), which plan selection binds to 1 when the -/// metric is declared a counter. `None` for any other estimate. -pub(crate) fn estimate_bound_over_raw_data_samples( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> Option { - let WeightDomain::RawDataSamplesOfMetric { metric } = &update.weight_domain else { - return None; - }; - let proven = SummaryUpdate { - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples, - }, - ..update.clone() - }; - if !estimate_has_bound(family, &proven, query) { - return None; - } - let _ = metric; - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters, its bound gated on the metric's sign") -} - -pub fn split_and(expr: &ScalarExpr, out: &mut Vec) { - match expr { - ScalarExpr::BoolAnd(items) => items.iter().for_each(|item| split_and(item, out)), - other => out.push(other.clone()), - } -} - -/// The columns `pred` pins to one literal value each: its conjuncts -/// `c = v` and `c IN (v)`. -fn pinned_columns(pred: &ScalarExpr) -> Vec { - let mut conjuncts = Vec::new(); - split_and(pred, &mut conjuncts); - conjuncts.iter().filter_map(pinned_column).collect() -} - -/// The column `conjunct` pins to one literal value (`c = v`, `c IN (v)`). -pub fn pinned_column(conjunct: &ScalarExpr) -> Option { - match conjunct { - ScalarExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - .. - } => match (&**left, &**right) { - (ScalarExpr::Column(column), ScalarExpr::Literal(_)) - | (ScalarExpr::Literal(_), ScalarExpr::Column(column)) => Some(*column), - _ => None, - }, - ScalarExpr::InList { - expr, - list, - negated: false, - } => match (&**expr, list.as_slice()) { - (ScalarExpr::Column(column), [ScalarExpr::Literal(_)]) => Some(*column), - _ => None, - }, - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::summary_type::{NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr}; - - /// A Count-Min estimate has a bound only over weights proven - /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. - #[test] - fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let summary_type = FieldDataType::Sketch(SketchAlgorithm::Cms); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let with = |weight_domain| SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain, - }; - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), - &query - )); - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::UnknownOrSigned), - &query - )); - assert!(estimate_has_bound( - &summary_type, - &with(WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }), - &query - )); - } -} diff --git a/crates/accuracy/src/propagation_stats.rs b/crates/accuracy/src/propagation_stats.rs deleted file mode 100644 index 58db409b9..000000000 --- a/crates/accuracy/src/propagation_stats.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Scoped source contracts and statistics the accuracy rules consult. -use asap_types::accuracy::GuaranteeSource; - -/// Statistics a propagation rule may consult. Every field is optional and -/// defaults to "unknown": a rule that needs a missing statistic emits a -/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct PropagationStats { - /// Provenance for supplied evidence (source, observation identity, etc.). - pub(crate) evidence_provenance: Vec, - /// Whether every input value is known to be non-negative — required by - /// the multiplicative relative-error rule, which is unsound across a - /// sign change. - pub(crate) values_non_negative: Option, - /// The statistic an unknown sign is named by when the folded values are - /// one metric's raw data samples or plain sums of them - /// ([`raw_data_samples_sign_statistic`](crate::composition::raw_data_samples_sign_statistic)): - /// the relative bound is then gated on it, and plan selection binds it - /// to 1 when the metric is declared a counter. `None`: the bare - /// `values_non_negative`, which nothing binds. - pub(crate) sign_statistic: Option, - /// Number of input rows an exact aggregation consumes (e.g. the number - /// of groups a function folds), for exact aggregate union bounds - /// bound over per-input failures. - pub(crate) input_row_count: Option, - /// The statistic an unknown row count is named by (default - /// `input_row_count`), so a fold whose rows the data workload's input - /// cardinality does not bound can leave a statistic nothing binds. - pub(crate) row_count_statistic: Option<&'static str>, - /// Lower confidence bound of the kth selected TopK item, after widening - /// the interval by the sketch's own estimation error. - pub(crate) topk_selected_lower_bound: Option, - /// Greatest upper confidence bound among excluded TopK items, after - /// widening the interval by the sketch's own estimation error. - pub(crate) topk_excluded_upper_bound: Option, - /// Union-bound failure probability of all intervals used by the margin - /// certificate. - pub(crate) topk_interval_failure_probability: Option, -} diff --git a/crates/accuracy/src/readout.rs b/crates/accuracy/src/readout.rs new file mode 100644 index 000000000..2ae234294 --- /dev/null +++ b/crates/accuracy/src/readout.rs @@ -0,0 +1,109 @@ +//! The metric of a readout's error: what a `SummaryEstimate` of a sketch +//! state may be off in. + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroup, Memo}; +use asap_types::summary_type::{parameter_list, SketchStatistic}; + +use crate::guarantee::guarantee_before_failure_probability_check; + +/// The metric of the error reading `statistic` from a state of +/// `summary_type` has: its first listed θ's guarantee's, whatever that +/// guarantee's failure probability (HLL's distinct count reads a relative +/// error although its failure probability is unknown). `None` for an exact +/// state and for a summary type with no listed θ or no guarantee for the +/// statistic (UnivMon's distinct count). +pub fn readout_metric( + summary_type: &FieldDataType, + statistic: &SketchStatistic, +) -> Option { + let FieldDataType::Sketch(algorithm) = summary_type else { + return None; + }; + let parameters = parameter_list(*algorithm).first()?; + Some( + guarantee_before_failure_probability_check(summary_type, parameters, statistic, 1.0)? + .spec + .accuracy_metric, + ) +} + +/// The metric of `candidate`'s error when it is a readout +/// (`SummaryEstimate`) of a sketch state ([`readout_metric`]); `None` for +/// any other candidate. +pub(crate) fn sketch_readout_metric( + memo: &Memo, + candidate: &CandidateForMemoGroup, +) -> Option { + let (summary_type, statistic) = sketch_readout(memo, candidate)?; + readout_metric(&summary_type, statistic) +} + +/// Per group, by group id, per candidate, by index in the group: its +/// [`sketch_readout_metric`]. +pub fn readout_metric_of_candidate(memo: &Memo) -> Vec>> { + memo.groups() + .iter() + .map(|group| { + group + .candidates + .iter() + .map(|candidate| sketch_readout_metric(memo, candidate)) + .collect() + }) + .collect() +} + +/// The summary type of the sketch state `candidate` reads and the statistic +/// it reads from it, when it is a readout (`SummaryEstimate`) of a sketch +/// state (`GroupProperties::sketch_algorithm_of_state`). +pub(crate) fn sketch_readout<'a>( + memo: &Memo, + candidate: &'a CandidateForMemoGroup, +) -> Option<(FieldDataType, &'a SketchStatistic)> { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) = &candidate.expression.operator + else { + return None; + }; + let algorithm = memo.groups()[summary_input.0 as usize] + .group_properties + .sketch_algorithm_of_state()?; + Some((FieldDataType::Sketch(algorithm), query)) +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::summary_type::{ExactKind, SketchAlgorithm}; + + /// A readout's metric is its algorithm's: KLL quantiles read a rank + /// error, DDSketch quantiles and HLL distinct counts a relative one + /// (HLL's although its failure probability is unknown); an exact state + /// reads exactly. + #[test] + fn a_readouts_metric_is_its_algorithms() { + let quantile = SketchStatistic::Quantile { q: 0.9 }; + let sketch = |algorithm| FieldDataType::Sketch(algorithm); + assert_eq!( + readout_metric(&sketch(SketchAlgorithm::Kll), &quantile), + Some(AccuracyMetric::Rank) + ); + assert_eq!( + readout_metric(&sketch(SketchAlgorithm::DDSketch), &quantile), + Some(AccuracyMetric::Relative) + ); + assert_eq!( + readout_metric(&sketch(SketchAlgorithm::Hll), &SketchStatistic::Cardinality), + Some(AccuracyMetric::Relative) + ); + assert_eq!( + readout_metric(&FieldDataType::exact(ExactKind::Sum), &quantile), + None + ); + } +} diff --git a/crates/accuracy/src/readout_caps.rs b/crates/accuracy/src/readout_caps.rs new file mode 100644 index 000000000..b0a3fff03 --- /dev/null +++ b/crates/accuracy/src/readout_caps.rs @@ -0,0 +1,446 @@ +//! Readout requirements (concepts doc, section 4, "Readout requirements" +//! and "Validity caps and pruning caps"): the caps the queries that reach +//! a readout pass down through their consumers' error contracts, and the +//! required-accuracy levels each readout gets from them. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::{Operator, QueryRoot}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy}; + +use crate::error_propagation; +use crate::readout::sketch_readout_metric; + +/// The error grid of the readouts of a query that combines several +/// ([`query_levels`]), loosest first. +const MAX_ERROR_LEVELS: [f64; 7] = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; + +/// The failure-probability grid of the readouts of a composed query: the +/// powers of ten from 1e-2 down to 1e-5, continued down to the first one at +/// or below a tenth of the query's per-value failure probability, so that up +/// to ten readouts each find a level and the levels sum within it (δ = 0.01 +/// over 10 000 series is 1e-6 per value: two quantiles of a ratio read at +/// 1e-7 each). +fn failure_probability_levels(per_value_failure_probability: f64) -> Vec { + let lowest = (per_value_failure_probability / 10.0).min(1e-5); + let deepest_exponent = (-lowest.log10() - 1e-9).ceil().min(300.0) as i32; + (2..=deepest_exponent) + .map(|exponent| 1.0 / 10f64.powi(exponent)) + .collect() +} + +/// Per group, by group id: `None` when no query with an accuracy target +/// reaches it; else per metric its output may be off in, per query, the +/// largest error in that metric the group's output may have for the query, +/// $\max_q \varepsilon_q / \lambda_{q,c,i}$ capped by the validity caps on +/// the way. A metric with no query has no cap: it reaches the group only +/// through inputs the output does not depend on ($\beta = 0$, a count's). +pub(crate) type Caps = Vec>>>; + +/// The required-accuracy levels of the sketch readouts +/// ([`sketch_readout_metric`]) without a requirement yet, per readout, with +/// the queries reaching it whose failure probability has no per-value one +/// there. `order` is the memo's topological order +/// (`Memo::topological_order`). +/// - A query whose path reads one readout group gives one level: its target +/// divided by the path's factor, the readout group's cap, at the target's +/// failure probability per readout value ([`query_levels`]). +/// - A composed query (several readout groups, such as a ratio) gives every +/// level of the grid [`MAX_ERROR_LEVELS`] × [`failure_probability_levels`] +/// not looser than the cap and the per-value failure probability (the cap +/// itself when no error level is), one δ level of 1 when the target sets +/// no failure probability. +/// - A readout group reached by several queries gets the union of their +/// levels, in query order; one no query with a target reaches in the +/// readout's metric gets none. Readouts of one group in one metric get +/// the same levels. +/// +/// A query whose target sets a failure probability that has no per-value +/// one at the readout (the values it covers or the readout group's values +/// per evaluation unknown) gives no level and is listed instead: Pass 2 +/// prunes a readout no query gives a level +/// (`RootPruneReason::ValuesPerEvaluationUnknown`), and lets a listed query +/// read the group only exactly when another query gives it a level. +#[allow(clippy::type_complexity)] +pub fn readout_levels( + memo: &Memo, + order: &[MemoGroupId], +) -> BTreeMap, Vec)> { + let caps = caps(memo, order); + let mut readouts: Vec<(CandidateForMemoGroupId, AccuracyMetric)> = Vec::new(); + for (index, group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in group.candidates.iter().enumerate() { + if candidate + .input_requirements + .first() + .is_some_and(|input| input.required_accuracy.is_none()) + { + if let Some(metric) = sketch_readout_metric(memo, candidate) { + let readout = CandidateForMemoGroupId { + group: MemoGroupId(index as u32), + index_in_group, + }; + readouts.push((readout, metric)); + } + } + } + } + let readout_groups: BTreeSet = + readouts.iter().map(|(readout, _)| readout.group).collect(); + let composed = composed_queries(memo, order, &readout_groups); + let mut levels_of_group: HashMap< + (MemoGroupId, AccuracyMetric), + (Vec, Vec), + > = HashMap::new(); + readouts + .into_iter() + .map(|(readout, metric)| { + let group = readout.group; + let levels = levels_of_group + .entry((group, metric)) + .or_insert_with(|| group_levels(memo, &caps, &composed, group, metric)) + .clone(); + (readout, levels) + }) + .collect() +} + +/// The levels of readout group `group` in `metric` ([`readout_levels`]), +/// with the queries reaching it without a per-value failure probability. +fn group_levels( + memo: &Memo, + caps: &Caps, + composed: &BTreeSet, + group: MemoGroupId, + metric: AccuracyMetric, +) -> (Vec, Vec) { + let readout_values = memo.series_count_scanned_below(group); + let mut levels: Vec = Vec::new(); + let mut queries_without_per_value_failure_probability = Vec::new(); + let group_caps = caps[group.0 as usize] + .as_ref() + .and_then(|caps| caps.get(&metric)) + .into_iter() + .flatten(); + for (&query, &cap) in group_caps { + let Some(target) = &memo.queries()[query].accuracy_target else { + continue; + }; + let Some(query_levels) = query_levels( + target, + memo.queries()[query].number_of_values_per_query_evaluation, + readout_values, + metric, + cap, + composed.contains(&query), + ) else { + queries_without_per_value_failure_probability.push(query); + continue; + }; + for level in query_levels { + if !levels.contains(&level) { + levels.push(level); + } + } + } + (levels, queries_without_per_value_failure_probability) +} + +/// The requirement levels one query gives a readout in `metric` whose cap +/// for it is `cap`; `None` when its target's failure probability has no +/// per-value one. A whole-result δ splits over the values one evaluation +/// returns, $n_q$; a reduction (`max`, `min`, `avg`) fails when any +/// readout it reads does, so δ also splits over the readout group's values +/// per evaluation, `readout_values`: the per-value failure probability is +/// $\delta / \max(n_q, \text{readout values})$ (`max(quantile_over_time(0.9, +/// m[5m]))` over 1000 series at δ = 0.01: 1e-5), and needs both known when +/// the target sets a δ. +fn query_levels( + target: &AccuracyTarget, + number_of_values_per_query_evaluation: Option, + readout_values: Option, + metric: AccuracyMetric, + cap: f64, + composed: bool, +) -> Option> { + let per_value_failure_probability = match target.failure_probability { + None => 1.0, + Some(delta) => { + let values_covered = target + .accuracy_scope + .max_values_covered(number_of_values_per_query_evaluation)?; + delta / values_covered.max(readout_values?).max(1) as f64 + } + }; + let level = |max_error, failure_probability| RequiredAccuracy { + metric, + max_error, + failure_probability, + }; + if !composed { + return Some(vec![level(cap, per_value_failure_probability)]); + } + let within = |grid: &[f64], limit: f64| -> Vec { + let kept: Vec = grid.iter().copied().filter(|&x| x <= limit).collect(); + if kept.is_empty() { + vec![limit] + } else { + kept + } + }; + let errors = within(&MAX_ERROR_LEVELS, cap); + let failure_probabilities = if target.failure_probability.is_none() { + vec![1.0] + } else { + within( + &failure_probability_levels(per_value_failure_probability), + per_value_failure_probability, + ) + }; + Some( + errors + .iter() + .flat_map(|&max_error| { + failure_probabilities + .iter() + .map(move |&failure_probability| level(max_error, failure_probability)) + }) + .collect(), + ) +} + +/// Each group's caps ([`Caps`]), one walk down from the query roots: each +/// query root group with a target is reached, with the target in each of +/// its metrics (`AccuracyTarget::metrics`); then, consumers before +/// producers, every candidate of a reached group reaches its inputs and +/// passes them each metric its error contract carries, with the caps +/// $\min(b_{c,i},\ \text{cap}/\beta_{c,i})$, keeping the largest per query +/// and metric (the bottleneck over paths); an input with $\beta_{c,i} = 0$ +/// gets the metric but no cap through that candidate. A scalar root's +/// operands are not reached. +pub(crate) fn caps(memo: &Memo, order: &[MemoGroupId]) -> Caps { + let mut caps: Caps = vec![None; memo.groups().len()]; + for (query, memo_query) in memo.queries().iter().enumerate() { + let (Some(target), QueryRoot::Operator(group)) = + (&memo_query.accuracy_target, &memo_query.root) + else { + continue; + }; + let group_caps = caps[group.0 as usize].get_or_insert_with(HashMap::new); + for metric in target.metrics() { + raise(group_caps, metric, query, target.max_error); + } + } + for &group in order.iter().rev() { + let Some(group_caps) = caps[group.0 as usize].clone() else { + continue; + }; + for consumer in &memo.groups()[group.0 as usize].candidates { + let operator = &consumer.expression.operator; + for &child in operator.children() { + caps[child.0 as usize].get_or_insert_with(HashMap::new); + } + for (&metric, caps_per_query) in &group_caps { + let Some(propagation) = error_propagation(memo, operator, metric) else { + continue; + }; + for (input, &child) in operator.children().into_iter().enumerate() { + let child_caps = caps[child.0 as usize].get_or_insert_with(HashMap::new); + child_caps.entry(metric).or_default(); + let factor = propagation.factor_per_input[input]; + // An input the output does not depend on (a count's, + // β = 0) gets no cap from it. + if factor > 0.0 { + let validity = + propagation.validity_cap_per_input[input].unwrap_or(f64::INFINITY); + for (&query, &cap) in caps_per_query { + raise(child_caps, metric, query, validity.min(cap / factor)); + } + } + } + } + } + } + caps +} + +/// The queries that combine several readouts: a candidate they reach has +/// two or more inputs below which a readout group lies (a ratio of two +/// quantiles). Readouts of one value by different summary types are +/// alternatives in one group's candidates, not a combination. +fn composed_queries( + memo: &Memo, + order: &[MemoGroupId], + readout_groups: &BTreeSet, +) -> BTreeSet { + let mut reads_a_readout = vec![false; memo.groups().len()]; + let mut composed = BTreeSet::new(); + for &group in order { + let candidates = &memo.groups()[group.0 as usize].candidates; + let inputs_reading = |operator: &Operator| { + operator + .children() + .into_iter() + .filter(|child| reads_a_readout[child.0 as usize]) + .count() + }; + let most_inputs_reading = candidates + .iter() + .map(|candidate| inputs_reading(&candidate.expression.operator)) + .max() + .unwrap_or(0); + if most_inputs_reading > 1 { + composed.extend( + memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .copied(), + ); + } + reads_a_readout[group.0 as usize] = + readout_groups.contains(&group) || most_inputs_reading > 0; + } + composed +} + +/// Keeps the larger of `cap` and the known cap of `query` in `metric`. +fn raise( + caps: &mut HashMap>, + metric: AccuracyMetric, + query: usize, + cap: f64, +) { + let known = caps.entry(metric).or_default().entry(query).or_insert(cap); + *known = known.max(cap); +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::operator::arguments::Reduction; + use asap_types::ir::operator::AggIntent; + use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; + use asap_types::stages::MemoQuery; + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; + use asap_types::test_support::memo_builders::{ + estimate, group, own_candidate, quantile_over_time, query, with_series, + }; + use asap_types::test_support::memo_fixtures::{aggregate, arithmetic}; + + /// The levels [`readout_levels`] gives the readout candidate `readout`. + fn levels_of(memo: &Memo, readout: MemoGroupId) -> Vec { + let order = memo.topological_order().expect("acyclic"); + readout_levels(memo, &order) + .remove(&own_candidate(readout)) + .expect("a sketch readout") + .0 + } + + /// A whole-result failure probability is split over the values one + /// evaluation returns: δ = 0.01 over 100 series is 1e-4 per value. + #[test] + fn a_whole_result_failure_probability_is_split_per_value() { + let mut memo = Memo::new(&with_series("m", 100)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: Some(100), + }); + let [required] = levels_of(&memo, readout)[..] else { + panic!("one requirement"); + }; + assert!((required.failure_probability - 1e-4).abs() < 1e-12); + } + + /// The snapshot workload + /// `ratio-of-quantiles-1h-mvp`: `quantile_over_time(0.9, data[1h]) / + /// quantile_over_time(0.5, data[1h])` with relative error 0.05 and + /// δ = 0.01 over the whole result of 10 000 series is δ = 1e-6 per + /// value, below the grid's 1e-5. The δ levels continue in powers of ten + /// below it, so both readouts find levels whose sum stays within 1e-6 + /// and (K) can hold. + #[test] + fn a_ratio_whose_per_value_failure_probability_is_below_the_grid_splits_it_over_its_readouts() { + let mut memo = Memo::new(&with_series("m", 10_000)); + let (_, state, p90) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 3600); + let p50 = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.5 }); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p90, p50)); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(ratio), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + number_of_values_per_query_evaluation: Some(10_000), + }); + let smallest_failure_probability = |readout| { + let levels = levels_of(&memo, readout); + assert!(!levels.is_empty(), "every readout has a level"); + levels + .into_iter() + .map(|required| required.failure_probability) + .fold(f64::INFINITY, f64::min) + }; + let (p90_delta, p50_delta) = ( + smallest_failure_probability(p90), + smallest_failure_probability(p50), + ); + assert!( + p90_delta + p50_delta <= 1e-6, + "{p90_delta} + {p50_delta} > 1e-6" + ); + } + + /// A `max` reads every readout of one evaluation, and fails when any of + /// them does: `max(quantile_over_time(0.9, m[5m]))` over 1000 series + /// with δ = 0.01 returns one value, yet each of its 1000 readouts must + /// fail at most 1e-5 for the union bound to hold. + #[test] + fn a_max_over_many_readouts_splits_the_failure_probability_over_them() { + let mut memo = Memo::new(&with_series("m", 1000)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + let max = group( + &mut memo, + aggregate(Reduction::by(vec![]), AggIntent::Max { col: None }, readout), + ); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(max), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + number_of_values_per_query_evaluation: Some(1), + }); + let levels = levels_of(&memo, readout); + assert!(!levels.is_empty()); + for level in levels { + assert!( + level.failure_probability <= 1e-5 * (1.0 + 1e-9), + "{level:?}" + ); + } + } + + /// A count reads no value (β = 0), so the readouts below it get no + /// level from it: `count(quantile_over_time(0.9, m[5m]))` requires + /// nothing of its readout, not an infinite error. + #[test] + fn a_count_over_readouts_gives_them_no_level() { + let mut memo = Memo::new(&with_series("m", 1000)); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + let count = group( + &mut memo, + aggregate(Reduction::by(vec![]), AggIntent::Count, readout), + ); + query( + &mut memo, + count, + Some(AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative)), + ); + assert_eq!(levels_of(&memo, readout), vec![]); + } +} diff --git a/crates/accuracy/src/requirements.rs b/crates/accuracy/src/requirements.rs deleted file mode 100644 index 58e7d78d6..000000000 --- a/crates/accuracy/src/requirements.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Whether a producer meets a consumer's logical requirements: -//! $\alpha^{\log}$ of a logical edge, and the limit an evaluated bound is -//! compared against. - -#[cfg(test)] -use asap_types::accuracy::AccuracyGuarantee; -use asap_types::stages::{CandidateForMemoGroupId, Memo}; - -use crate::AccuracyEvidence; - -/// $\alpha^{\log}$ of one edge (concepts doc, section 2): for a readout's -/// input, the producer's guarantee for the readout's statistic at the -/// required failure probability is within the required error, and a -/// top-k's heap holds k; for any other input, the metric rule (section 4). -pub fn producer_satisfies_logical_requirements( - memo: &Memo, - producer: CandidateForMemoGroupId, - consumer: CandidateForMemoGroupId, - input_index_of_consumer: usize, - evidence: &dyn AccuracyEvidence, -) -> bool { - let _ = (memo, producer, consumer, input_index_of_consumer, evidence); - todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") -} - -/// Small relative tolerance for comparing an evaluated bound against a -/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected -/// by floating-point noise. -const SATISFACTION_TOLERANCE: f64 = 1e-9; - -/// The largest evaluated error (or failure probability) that meets a -/// required `limit`: `limit` widened by [`SATISFACTION_TOLERANCE`], so a -/// bound equal to it up to floating-point noise meets it. -pub fn target_limit(limit: f64) -> f64 { - limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON -} - -/// Whether `guarantee`'s evaluated error is within `max_error` and, when -/// `failure_probability` is set, its evaluated failure probability within -/// it ([`target_limit`]); unknown values do not meet. -#[cfg(test)] -pub(crate) fn meets( - guarantee: &AccuracyGuarantee, - max_error: f64, - failure_probability: Option, -) -> bool { - let within = |value: Option, limit: f64| value.is_some_and(|v| v <= target_limit(limit)); - within(guarantee.spec.error.evaluate(), max_error) - && failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A bound equal to the limit up to floating-point noise meets it; a - /// bound one part in a million above it does not. - #[test] - fn target_limit_admits_only_floating_point_noise() { - assert!(0.01 * (1.0 + 1e-12) <= target_limit(0.01)); - assert!(0.01 * (1.0 + 1e-6) > target_limit(0.01)); - assert!(0.0 <= target_limit(0.0)); - } -} diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs deleted file mode 100644 index a79d8b5c2..000000000 --- a/crates/accuracy/src/sizing.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! The statistic a summary is read for. (The predefined parameter lists -//! are checked against the closed-form sizing for an `(ε, δ)` in -//! `guarantee::sizing_oracle`, which is test-only.) - -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::ColumnRef; -use asap_types::summary_type::SketchStatistic; - -/// The statistic a summary is read for to answer `intent`, or `None` when -/// no sketch statistic serves the intent. -pub fn sketch_statistic(intent: &AggIntent) -> Option { - Some(match intent { - AggIntent::Quantile { q, .. } => SketchStatistic::Quantile { q: *q }, - AggIntent::Cardinality { .. } => SketchStatistic::Cardinality, - AggIntent::FrequencyL2 { .. } => SketchStatistic::FrequencyL2, - AggIntent::FrequencyEntropy { .. } => SketchStatistic::FrequencyEntropy, - AggIntent::TopK { k, .. } => SketchStatistic::TopK { k: *k }, - AggIntent::Count => SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - _ => return None, - }) -} diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index adc4dc5ab..524548bba 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -778,11 +778,9 @@ fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", OriginOfCandidateForMemoGroup::TopKReuse => "top_k_reuse", - OriginOfCandidateForMemoGroup::WidenedScans => "widened_scans", } .into() } diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 160d792db..e6faccf28 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -1,7 +1,6 @@ //! The planner stages' output as plain text: `asap_planner --format text`. //! It prints, up to the run's last stage, the frontends' roots and their -//! memo groups (Stage 0), Pass 1's `Memo`, -//! the memo after Pass 2's local window composition and Pass 2's +//! memo groups (Stage 0), Pass 1's `Memo`, Pass 2's //! `LogicalASAPMemoCandidates` (Stage 2's input), Stage 2's //! `PhysicalASAPMemoCandidates` (Stage 3's input), and Stage 3's //! `Selection` with the selected memo choice, each node's cost and each diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index f9c3068dc..bf8a710a5 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -79,10 +79,7 @@ const SECTIONS: [(&str, &[&str]); 5] = [ ("stage1_pass1", &["Stage 1 Pass 1 output: Memo"]), ( "stage1_pass2", - &[ - "Stage 1 Pass 2, after local window composition", - "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", - ], + &["Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates"], ), ( "stage2", diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index 4b4d58ff1..8863e035f 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -34,8 +34,7 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result>) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 0696d2816..dc7726b2b 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -58,7 +58,6 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { lifetime: Lifetime::None, parameters: None, output_schema: output, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index e730108e4..f262878ae 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -160,7 +160,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { }, ])) .clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index b77a7587b..ec9432e08 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -532,7 +532,6 @@ fn bind_physical_asap_dag_before_execution() { lifetime: Default::default(), parameters: None, output_schema: (*schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -694,7 +693,6 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1116,7 +1114,6 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { lifetime: Default::default(), parameters: None, output_schema: (*input).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1208,7 +1205,6 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { kind: JoinKind::Semi, @@ -1316,7 +1312,6 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { operator: BinaryOperator { diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 1413688fb..79be1b280 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -387,7 +387,6 @@ fn global_extrema_bind_with_planner_derived_schema() { lifetime: Default::default(), parameters: None, output_schema: (*output).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -931,7 +930,6 @@ fn planner_node( }) .collect(), ), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -1165,7 +1163,6 @@ fn exact_aggregate_node( lifetime: Default::default(), parameters: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 64f64962f..a02882402 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -52,7 +52,6 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 938cf0b5a..39486bb2d 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -49,7 +49,6 @@ fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode lifetime: Default::default(), parameters: None, output_schema, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -138,7 +137,6 @@ fn try_state_dag( ), parameters: None, output_schema: logical_schema(family.clone()), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -154,7 +152,6 @@ fn try_state_dag( lifetime: Default::default(), parameters: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -175,7 +172,6 @@ fn try_state_dag( ), parameters: None, output_schema: logical_schema(target), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index d2c03d033..fb36382d1 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -82,7 +82,6 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { lifetime: Default::default(), parameters: None, output_schema: (*schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -422,7 +421,6 @@ fn stored_series_evaluations_support_filters_and_sets() { 5 | 6 => sample_schema(), _ => value_schema.clone(), }, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 10bc11ffe..25d3fe675 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -87,7 +87,6 @@ fn plan( lifetime: Default::default(), parameters: None, output_schema: (**schema).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -510,7 +509,6 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -845,7 +843,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { lifetime: Default::default(), parameters: None, output_schema: output_schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 4c385fca6..9570e9fcb 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -318,7 +318,6 @@ fn merge_over_a_stored_tumbling_window( output_state, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index c624a6d9e..48b458694 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -55,7 +55,6 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { output_state: ExecutionDataState::QUERY_ROWS, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -126,7 +125,6 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -156,7 +154,6 @@ fn summary_projection_survives_recovery() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -265,7 +262,6 @@ fn summary_merge_without_inputs_is_rejected() { output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: Default::default(), parameters: None, - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 49241d6a5..fcc1ba2fd 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -181,7 +181,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() lifetime: Default::default(), parameters: None, output_schema: (*state).clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index bc74bcd3d..f27c0be72 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -21,6 +21,8 @@ mod operator_design_examples; mod operator_sharing; mod pass1_candidates; mod pass1_candidates_sql; +mod pass2_accuracy_and_grids_promql; +mod pass2_rollup_promql; mod per_series_exact_execution; mod production_rollup_workloads; mod promql_to_post_asap; diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index 53ef51dfd..5c762eb6c 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json index f822d6b70..f0abf8bbe 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json @@ -1,10 +1,10 @@ [ -{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, -{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, -{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, -{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, -{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, -{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} +{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, +{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, +{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, +{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, +{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, +{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} ] diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 80ceb6abf..654c8dc74 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -83,15 +83,10 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { })) .unwrap(), ); - let finalize = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - }), - node.schema.clone(), - ) - .with_guarantee(None), - ); + let finalize = std::rc::Rc::new(OperatorNode::with_schema( + asap_types::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + node.schema.clone(), + )); return finalize; } Rc::new(node.with_new_children(rewrite).unwrap()) diff --git a/crates/integration-tests/tests/pass1_candidates_sql.rs b/crates/integration-tests/tests/pass1_candidates_sql.rs index 17db96566..d4d0f3600 100644 --- a/crates/integration-tests/tests/pass1_candidates_sql.rs +++ b/crates/integration-tests/tests/pass1_candidates_sql.rs @@ -1,5 +1,5 @@ -//! SQL → Pass 1 memo candidates: where each `HAVING` is applied, the -//! guarantees composed over estimates, and what a Top-K heap sketch ranks. +//! SQL → Pass 1 memo candidates: where each `HAVING` is applied and what a +//! Top-K heap sketch ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_stage1_pass1_per_computation_candidates::pass1_memo; diff --git a/crates/integration-tests/tests/pass2_accuracy_and_grids_promql.rs b/crates/integration-tests/tests/pass2_accuracy_and_grids_promql.rs new file mode 100644 index 000000000..91c350c30 --- /dev/null +++ b/crates/integration-tests/tests/pass2_accuracy_and_grids_promql.rs @@ -0,0 +1,148 @@ +//! PromQL → Pass 2's error contracts and evaluation grids, through +//! production lowering: the frontend's `Project`s of a function of the +//! value keep a readout's error bounded, and a subquery's resolution is the +//! evaluation grid of the groups below it. +use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; +use asap_planner::PlanningModels; +use asap_types::ir::operator::asap::summary_type_of_operator; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, MemoGroupId}; +use asap_types::workload::PlanningWorkload; +use serde_json::{json, Value}; + +/// Pass 2's output for `query`, repeating every minute at `accuracy`, over +/// the gauge `m` and the counter `c`, 1000 series each. +fn pass2(query: &str, accuracy: Value) -> LogicalASAPMemoCandidates { + let workload: PlanningWorkload = serde_json::from_value(json!({ + "query_workload": { + "language": "promql", + "repeating_queries": [{ + "query": query, + "demand": {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}, + "requirements": {"accuracy": accuracy}, + "predictability": {"predictable": {"known_at": null}}, + }], + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": { + "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, + "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + }, + }, + })) + .unwrap(); + workload.validate().unwrap(); + let roots = lower_promql_roots(&workload).unwrap(); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_stages_until( + roots, + &entries, + &data, + PlanningModels::builtin(), + RunUntil::Pass2, + ) + .unwrap(); + run.run.logical_asap_candidates.unwrap() +} + +/// The sketch readouts (`SummaryEstimate` over a sketch state) that +/// survive Pass 2. +fn live_sketch_readouts(logical: &LogicalASAPMemoCandidates) -> usize { + let memo = &logical.memo; + let mut live = 0; + for (group, memo_group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = + &candidate.expression.operator + else { + continue; + }; + let sketch = matches!( + summary_type_of_operator(&memo.groups()[summary_input.0 as usize].expression), + Some(FieldDataType::Sketch(_)) + ); + let id = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + if sketch && logical.pruned.iter().all(|pruned| pruned.candidate != id) { + live += 1; + } + } + } + live +} + +/// The frontend lowers `abs(v)`, `-v` and `v / 2` to a `Project` that +/// replaces the value and passes the series identity through +/// `promql_drop_metric_name`. Its contract judges only the value: `abs` +/// and the negation keep an absolute error, scaling by a constant a +/// relative one, so a sketch readout below each still serves the query, as +/// it does for the bare readout. +#[test] +fn a_function_of_the_value_keeps_the_readouts_below_it() { + let absolute = json!({"max_error": 0.05, "accuracy_metric": "absolute"}); + let relative = json!({"max_error": 0.05, "accuracy_metric": "relative"}); + for (query, accuracy) in [ + ("count_over_time(m[1m])", &absolute), + ("abs(count_over_time(m[1m]))", &absolute), + ("-count_over_time(m[1m])", &absolute), + ("quantile_over_time(0.9, m[5m]) / 2", &relative), + ("quantile_over_time(0.9, m[5m]) * 2", &relative), + ] { + let logical = pass2(query, accuracy.clone()); + assert!(live_sketch_readouts(&logical) > 0, "{query}"); + } +} + +/// The groups whose expression is a tumbling-window chain, with its width. +fn chain_widths(logical: &LogicalASAPMemoCandidates) -> Vec { + logical + .memo + .groups() + .iter() + .filter_map(|group| match group.expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + }) + .collect() +} + +/// `quantile_over_time(0.9, rate(c[2m])[10m:30s])` read every 60 s +/// evaluates the inner `rate` every 30 s: its groups are on the 30 s grid +/// and read only chains whose width divides 30 s. Without a resolution the +/// inner step is the server's default, unknown: no grid, no chain. +#[test] +fn a_subquery_s_resolution_is_the_grid_of_the_groups_below_it() { + let rank = json!({"max_error": 0.01, "accuracy_metric": "rank"}); + let logical = pass2( + "quantile_over_time(0.9, rate(c[2m])[10m:30s])", + rank.clone(), + ); + let subquery = logical + .memo + .groups() + .iter() + .position(|group| { + matches!( + group.expression, + Operator::NonASAP(NonASAPOp::PromqlSubquery { .. }) + ) + }) + .expect("a subquery group"); + let child = logical.memo.groups()[subquery].expression.children()[0].0 as usize; + let period = |group: usize| logical.evaluation_grid_of_group[group].map(|grid| grid.period_ms); + assert_eq!(period(subquery), Some(60_000)); + assert_eq!(period(child), Some(30_000)); + let widths = chain_widths(&logical); + assert!(!widths.is_empty()); + assert!(widths.iter().all(|width| 30_000 % width == 0), "{widths:?}"); + let without_resolution = pass2("max_over_time(rate(c[2m])[10m:])", rank); + assert_eq!(chain_widths(&without_resolution), Vec::::new()); +} diff --git a/crates/integration-tests/tests/pass2_rollup_promql.rs b/crates/integration-tests/tests/pass2_rollup_promql.rs new file mode 100644 index 000000000..22a1e7133 --- /dev/null +++ b/crates/integration-tests/tests/pass2_rollup_promql.rs @@ -0,0 +1,160 @@ +//! PromQL → Pass 2's rollup rule, through production lowering: a coarse +//! cross-series aggregate is offered from a finer one over the same +//! per-series computation, although the two queries' scans list different +//! labels and an instant or range `TimeRange` (and a per-series function) +//! sits between each scan and its aggregate. +use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; +use asap_planner::PlanningModels; +use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; +use asap_types::workload::PlanningWorkload; +use serde_json::json; + +/// Pass 2's memo of `queries`, each repeating every minute at a 1% error +/// target, over the gauge `m`. +fn pass2_memo(queries: &[&str]) -> Memo { + let workload: PlanningWorkload = serde_json::from_value(json!({ + "query_workload": { + "language": "promql", + "repeating_queries": queries.iter().map(|query| json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.01}}, + "predictability": {"predictable": {"known_at": null}}, + })).collect::>(), + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}}, + }, + })) + .unwrap(); + workload.validate().unwrap(); + let roots = lower_promql_roots(&workload).unwrap(); + let entries: Vec<_> = workload.query_workload.entries().collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let run = run_stages_until( + roots, + &entries, + &data, + PlanningModels::builtin(), + RunUntil::Pass2, + ) + .unwrap(); + run.run.logical_asap_candidates.unwrap().memo +} + +/// The root group of query `query`. +fn root(memo: &Memo, query: usize) -> MemoGroupId { + match memo.queries()[query].root { + QueryRoot::Operator(group) => group, + ref other => panic!("query {query} has no operator root: {other:?}"), + } +} + +/// The groups the rollup candidates of `group` read: the finer group +/// itself, or the finer group under the compensating `Filter`. +fn rolled_up_from(memo: &Memo, group: MemoGroupId) -> Vec { + memo.groups()[group.0 as usize] + .candidates + .iter() + .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|candidate| { + let [input] = candidate.expression.operator.children()[..] else { + panic!("a rollup reads one input: {:?}", candidate.expression); + }; + match &memo.groups()[input.0 as usize].expression { + Operator::NonASAP(NonASAPOp::Filter { child, .. }) => *child, + _ => *input, + } + }) + .collect() +} + +/// Every rollup candidate of the memo. +fn rollup_count(memo: &Memo) -> usize { + memo.groups() + .iter() + .flat_map(|group| &group.candidates) + .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .count() +} + +#[test] +fn an_instant_sum_rolls_up_from_the_finer_sum_over_a_scan_listing_more_labels() { + let memo = pass2_memo(&["sum by (job) (m)", "sum by (job, instance) (m)"]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); + assert!(rolled_up_from(&memo, root(&memo, 1)).is_empty()); +} + +/// A count rolls up as a sum of the finer counts, under the coarse +/// count's own column name (`count`, where the sum's would be `sum`). +#[test] +fn an_instant_count_rolls_up_from_the_finer_count() { + let memo = pass2_memo(&["count by (job) (m)", "count by (job, instance) (m)"]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); +} + +#[test] +fn a_sum_of_rates_rolls_up_from_the_finer_sum_of_the_same_rates() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (rate(m[5m]))", + ]); + assert_eq!(rolled_up_from(&memo, root(&memo, 0)), vec![root(&memo, 1)]); +} + +#[test] +fn a_label_matcher_becomes_a_compensating_filter_over_the_finer_sum() { + let memo = pass2_memo(&[ + r#"sum by (job) (m{instance="i1"})"#, + "sum by (job, instance) (m)", + ]); + let candidates = &memo.groups()[root(&memo, 0).0 as usize].candidates; + let rollup = candidates + .iter() + .find(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup) + .expect("a rollup candidate"); + let [filter] = rollup.expression.operator.children()[..] else { + panic!("a rollup reads one input"); + }; + let Operator::NonASAP(NonASAPOp::Filter { child, .. }) = + &memo.groups()[filter.0 as usize].expression + else { + panic!("the rollup reads a compensating filter"); + }; + assert_eq!(*child, root(&memo, 1)); +} + +#[test] +fn rates_over_different_ranges_do_not_roll_up() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (rate(m[10m]))", + ]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn different_per_series_functions_do_not_roll_up() { + let memo = pass2_memo(&[ + "sum by (job) (rate(m[5m]))", + "sum by (job, instance) (increase(m[5m]))", + ]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn a_standard_deviation_does_not_roll_up() { + let memo = pass2_memo(&["stddev by (job) (m)", "stddev by (job, instance) (m)"]); + assert_eq!(rollup_count(&memo), 0); +} + +#[test] +fn a_finer_sum_narrowed_by_a_matcher_serves_no_wider_sum() { + let memo = pass2_memo(&[ + "sum by (job) (m)", + r#"sum by (job, instance) (m{instance="i1"})"#, + ]); + assert!(rolled_up_from(&memo, root(&memo, 0)).is_empty()); +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs index a8e5e2689..330a8a651 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/measure_output.rs @@ -13,9 +13,7 @@ use asap_types::ir::{ ProjectItem, ScalarExpr, }; use asap_types::stages::{Memo, MemoGroupId}; -use asap_types::summary_type::{ExactKind, SummaryInputExpr}; - -use asap_accuracy::sizing::sketch_statistic; +use asap_types::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr}; use super::pending::{add_candidate, commit_pending, Pending}; use super::SingleMeasure; @@ -180,9 +178,9 @@ pub(super) fn summary_readout( reduction: &Reduction, result_group: MemoGroupId, ) -> Result, PerComputationCandidateError> { - let query = sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( - "sketch evaluation", - ))?; + let query = SketchStatistic::for_intent(intent).ok_or( + PerComputationCandidateError::Unsupported("sketch evaluation"), + )?; let state = pending.push(memo, build)?; let estimate = Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: state, diff --git a/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs b/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs index 44fd9d99e..e375abdb2 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/candidates/pending.rs @@ -4,14 +4,13 @@ //! check that can find the way unsupported has passed, so an unsupported //! way adds nothing. -use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::Schema; use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ - CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, - MemoGroupId, OriginOfCandidateForMemoGroup, + listed_build_parameters, CandidateForMemoGroup, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, }; -use asap_types::summary_type::{parameter_list, SketchParams}; +use asap_types::summary_type::SketchParams; use crate::{schema, PerComputationCandidateError}; @@ -135,18 +134,7 @@ pub(crate) fn add_state_group( fused_groups: &[MemoGroupId], ) -> Result { let group = memo.find_or_add_group(build.clone())?; - let configurations: Vec> = match &build { - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(algorithm), - .. - }) => parameter_list(*algorithm) - .iter() - .cloned() - .map(Some) - .collect(), - _ => vec![None], - }; - for parameters in configurations { + for parameters in listed_build_parameters(&build) { add_candidate( memo, group, @@ -168,19 +156,14 @@ pub(crate) fn add_candidate( parameters: Option, fused_groups: Vec, ) -> Result<(), PerComputationCandidateError> { - let expression = CandidateExpression { operator }; - let logical_properties = LogicalProperties::new(&expression, parameters)?; - let input_requirements = - vec![LogicalRequirements::default(); expression.operator.children().len()]; memo.add_candidate( group, - CandidateForMemoGroup { - expression, - logical_properties, - input_requirements, + CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, fused_groups, - origin: OriginOfCandidateForMemoGroup::Pass1, - }, + ), )?; Ok(()) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index 91839069c..7871028e6 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -23,10 +23,9 @@ use asap_types::ir::schema::{ColumnId, Schema}; use asap_types::ir::{NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr}; use asap_types::stages::{Memo, MemoGroupId}; use asap_types::summary_type::{ - ExactKind, SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, + ExactKind, SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, }; -use asap_accuracy::sizing::sketch_statistic; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; use crate::summary_realization::{realize_keyed_additive_summary_input, summary_candidates}; @@ -117,7 +116,7 @@ pub(crate) fn computation_candidates_for_intent( } let quantile_outside_unit_interval = matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)); - if sketch_statistic(intent).is_some() && !quantile_outside_unit_interval { + if SketchStatistic::for_intent(intent).is_some() && !quantile_outside_unit_interval { choices.extend( summary_candidates(intent) .into_iter() @@ -251,7 +250,8 @@ fn without_unsupported_summaries( else { continue; }; - let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) + let Some(readout) = + SketchStatistic::for_intent(&measures[target.measure_index]).map(|s| s.kind()) else { continue; }; diff --git a/crates/stage1-pass1-per-computation-candidates/src/lib.rs b/crates/stage1-pass1-per-computation-candidates/src/lib.rs index 0effe69fe..99eb6c8fd 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/lib.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/lib.rs @@ -37,9 +37,7 @@ mod tests; use asap_types::deployment_capabilities::SupportedSummary; use asap_types::ir::schema::Schema; use asap_types::ir::{Operator, SchemaDerivationError}; -use asap_types::stages::{ - CandidateError, LogicalPropertiesError, Memo, MemoGroupError, MemoGroupId, -}; +use asap_types::stages::{CandidateError, Memo, MemoGroupError, MemoGroupId}; use thiserror::Error; use candidates::add_pass1_candidates; @@ -62,10 +60,6 @@ pub enum PerComputationCandidateError { /// its group's result: a bug in the rule that built it. #[error(transparent)] Candidate(#[from] CandidateError), - /// A candidate Pass 1 built has logical properties that cannot go - /// together: a bug in the rule that built it. - #[error(transparent)] - LogicalProperties(#[from] LogicalPropertiesError), #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] Unsupported(&'static str), } diff --git a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs index 4d86fa54a..f78185da3 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/summary_realization.rs @@ -15,12 +15,10 @@ use asap_types::stages::{ ranked_column_of_group, raw_data_samples_of_metric_of_group, Memo, MemoGroupId, }; use asap_types::summary_type::{ - EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, + EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, }; -use asap_accuracy::sizing::sketch_statistic; - use crate::computation_targets::nested_aggregates; use crate::{expression, schema, PerComputationCandidateError}; @@ -45,7 +43,7 @@ const ALGORITHM_PREFERENCE: &[SketchAlgorithm] = &[ /// count. Empty for an intent no sketch /// statistic serves. pub(crate) fn summary_candidates(intent: &AggIntent) -> Vec { - let Some(statistic) = sketch_statistic(intent) else { + let Some(statistic) = SketchStatistic::for_intent(intent) else { return Vec::new(); }; // A distinct-tuple count hashes the whole tuple as one item diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index bb0ec22e0..3102f543a 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -1,38 +1,13 @@ -//! Fixtures this crate's own tests share. The memo builders add Stage 0's -//! groups through `Memo::find_or_add_group` and `Memo::add_query`, as -//! Stage 0 does, so no test calls another stage crate. The expressions -//! come from `asap_types::test_support::memo_fixtures`; its scans list Utf8 -//! labels. +//! Fixtures this crate's own tests share, beside the memo builders of +//! `asap_types::test_support::memo_builders` (`group`, `scan_group`, +//! `query`), so no test calls another stage crate. use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::series_identity::with_promql_series_identity; -use asap_types::ir::schema::DataType; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot}; -use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; -use asap_types::test_support::memo_fixtures::{aggregate_with, scan}; - -/// The group of `operator`, added as Stage 0 adds it. -pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { - memo.find_or_add_group(operator).unwrap() -} - -/// The group of a `TimeSeries(metric)` scan listing `labels`, its series -/// identity realized as the planner realizes it before Stage 0 -/// (`with_promql_series_identity`): a closed schema whose last column is -/// `$promql_series_identity`, keyed by it and the timestamp. -pub fn scan_group(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { - let labels: Vec<(&str, DataType)> = labels.iter().map(|l| (*l, DataType::Utf8)).collect(); - let mut operator = scan(metric, &labels, vec![]); - let open = - OperatorNode::new_shared(operator.map_children(|_| unreachable!("a scan has no input"))) - .unwrap(); - let realized = with_promql_series_identity(&open).unwrap().schema.clone(); - if let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = &mut operator { - *schema = realized; - } - group(memo, operator) -} +use asap_types::ir::Predicate; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::test_support::memo_builders::group; +use asap_types::test_support::memo_fixtures::aggregate_with; /// The group of an `Aggregate` of `measures` (unnamed, unfiltered) over /// `child`, grouped by `reduction`, with `having`. @@ -46,12 +21,3 @@ pub fn aggregate_group( let measures = measures.into_iter().map(|m| (m, None)).collect(); group(memo, aggregate_with(reduction, measures, having, child)) } - -/// Adds a query whose root is group `root`. -pub fn query(memo: &mut Memo, root: MemoGroupId) -> usize { - memo.add_query(MemoQuery { - root: QueryRoot::Operator(root), - accuracy_target: None, - number_of_values_per_query_evaluation: None, - }) -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/tests.rs b/crates/stage1-pass1-per-computation-candidates/src/tests.rs index 272151b8a..460cb6351 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/tests.rs @@ -27,8 +27,9 @@ use crate::computation_targets::{ use crate::summary_realization::{ realize_keyed_additive_summary_input, summarised_input, summary_candidates, }; -use crate::test_fixtures::{aggregate_group, group, query, scan_group}; +use crate::test_fixtures::aggregate_group; use crate::{pass1_memo, PerComputationCandidateError}; +use asap_types::test_support::memo_builders::{group, query, scan_group}; use asap_types::test_support::memo_fixtures::scan; use asap_types::workload::DataWorkload; @@ -138,7 +139,7 @@ fn topk(k: usize) -> AggIntent { /// `metric[5m]` (`Range`) or `metric` (`Instant`, which a cross-series /// aggregate reads): a selector over `metric{labels}`. fn selector(memo: &mut Memo, kind: TimeRangeKind, metric: &str, labels: &[&str]) -> MemoGroupId { - let scan = scan_group(memo, metric, labels); + let scan = scan_group(memo, metric, labels, vec![]); group( memo, Operator::NonASAP(NonASAPOp::TimeRange { @@ -211,8 +212,8 @@ fn running_example_gives_one_build_per_theta_and_separate_readouts() { rhs: g5, }), ); - query(&mut memo, g3); - query(&mut memo, g6); + query(&mut memo, g3, None); + query(&mut memo, g6, None); let memo = pass1_memo(memo, None).unwrap(); println!("{}", memo_printout(&memo)); let (kll, _) = state_group(&memo, SketchAlgorithm::Kll); @@ -276,7 +277,7 @@ fn per_series_readouts_keep_the_series_identity() { vec![quantile(0.9)], None, ); - query(&mut memo, per_series); + query(&mut memo, per_series, None); let memo = pass1_memo(memo, None).unwrap(); let output = &memo.groups()[per_series.0 as usize].group_properties.schema; assert!(output.closed && output.has_promql_series_identity()); @@ -329,7 +330,7 @@ fn an_open_per_series_aggregate_offers_no_renamed_readout() { vec![quantile(0.9)], None, ); - query(&mut memo, per_series); + query(&mut memo, per_series, None); let memo = pass1_memo(memo, None).unwrap(); assert!( !memo.groups()[per_series.0 as usize] @@ -355,7 +356,7 @@ fn multi_measure_aggregate_has_one_target_per_measure() { vec![sum(), quantile(0.9)], None, ); - query(&mut memo, both); + query(&mut memo, both, None); let targets = computation_targets(&memo, None).unwrap(); let measures: Vec<(MemoGroupId, usize)> = targets .iter() @@ -380,7 +381,7 @@ fn aggregate_without_realization_is_not_a_target() { }], None, ); - query(&mut memo, stddev); + query(&mut memo, stddev, None); assert!(computation_targets(&memo, None).unwrap().is_empty()); let memo = pass1_memo(memo, None).unwrap(); assert_eq!(candidates(&memo, stddev).len(), 1); @@ -399,8 +400,8 @@ fn an_aggregate_read_by_several_roots_is_one_target() { vec![quantile(0.9)], None, ); - query(&mut memo, p); - query(&mut memo, p); + query(&mut memo, p, None); + query(&mut memo, p, None); assert_eq!(computation_targets(&memo, None).unwrap().len(), 1); } @@ -420,7 +421,7 @@ fn asap_input_is_rejected() { filter: None, }), ); - query(&mut memo, state); + query(&mut memo, state, None); assert!(matches!( pass1_memo(memo, None), Err(PerComputationCandidateError::ContainsAsap) @@ -439,7 +440,7 @@ fn unsupported_summaries_are_not_offered() { vec![quantile(0.9)], None, ); - query(&mut memo, p); + query(&mut memo, p, None); let memo = pass1_memo(memo, Some(&[])).unwrap(); assert_eq!(candidates(&memo, p).len(), 1); assert!(groups_where(&memo, |e| matches!(e, Operator::ASAP(_))).is_empty()); @@ -454,7 +455,7 @@ fn every_group_offers_its_own_expression() { let mut memo = Memo::new(&DataWorkload::default()); let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, s); + query(&mut memo, s, None); let stage0 = memo.groups().len(); let memo = pass1_memo(memo, None).unwrap(); for index in 0..stage0 { @@ -473,7 +474,7 @@ fn exact_accumulator_is_a_state_group_and_a_finalize() { let mut memo = Memo::new(&DataWorkload::default()); let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, s); + query(&mut memo, s, None); let memo = pass1_memo(memo, None).unwrap(); let states = groups_where(&memo, |e| { matches!( @@ -539,7 +540,7 @@ fn multi_measure_candidate_joins_single_measure_groups() { ], None, ); - query(&mut memo, both); + query(&mut memo, both, None); let memo = pass1_memo(memo, None).unwrap(); let single = groups_where( &memo, @@ -585,8 +586,8 @@ fn pass1_sets_no_input_requirement() { vec![quantile(0.9)], None, ); - query(&mut memo, ranked); - query(&mut memo, p); + query(&mut memo, ranked, None); + query(&mut memo, p, None); let memo = pass1_memo(memo, None).unwrap(); // The top-k readout is offered, and has no input requirement either. assert!(!groups_where(&memo, |e| matches!( @@ -641,8 +642,8 @@ fn key_only_having_is_pushed_below_into_the_filter_group() { by_job, compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())), ); - query(&mut memo, having); - query(&mut memo, by_job); + query(&mut memo, having, None); + query(&mut memo, by_job, None); let memo = pass1_memo(memo, None).unwrap(); let below = pushed_below(&memo, rows); assert_eq!(below.len(), 1, "{}", memo_printout(&memo)); @@ -670,7 +671,7 @@ fn value_having_stays_above() { by_job, compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), ); - query(&mut memo, having); + query(&mut memo, having, None); let memo = pass1_memo(memo, None).unwrap(); assert!(pushed_below(&memo, rows).is_empty()); assert_eq!(candidates(&memo, having).len(), 1); @@ -694,7 +695,7 @@ fn global_aggregate_having_stays_above() { p, Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))), ); - query(&mut memo, having); + query(&mut memo, having, None); let memo = pass1_memo(memo, None).unwrap(); assert!(pushed_below(&memo, rows).is_empty()); } @@ -760,7 +761,7 @@ fn a_constant_having_of_a_grouped_aggregate_is_pushed_below() { semantics: ExprSemantics::Sql, }), ); - query(&mut memo, having); + query(&mut memo, having, None); let memo = pass1_memo(memo, None).unwrap(); assert_eq!( pushed_below(&memo, rows).len(), @@ -784,7 +785,7 @@ fn own_value_having_filters_each_readout() { vec![quantile(0.9)], Some(compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0))), ); - query(&mut memo, by_job); + query(&mut memo, by_job, None); let memo = pass1_memo(memo, None).unwrap(); let kinds: Vec<&str> = candidates(&memo, by_job) .iter() @@ -820,7 +821,7 @@ fn whole_expression_topk_heaps_read_the_raw_samples() { vec![topk(5)], None, ); - query(&mut memo, ranked); + query(&mut memo, ranked, None); let memo = pass1_memo(memo, None).unwrap(); for algorithm in [ SketchAlgorithm::CmsWithHeap, @@ -875,7 +876,7 @@ fn whole_expression_topk_heap_candidates() { vec![topk(5)], None, ); - query(&mut memo, ranked); + query(&mut memo, ranked, None); let memo = pass1_memo(memo, None).unwrap(); // The top-k group's own expression, then one ranked `Project` per heap. let ranked_candidates = candidates(&memo, ranked); @@ -938,7 +939,7 @@ fn count_ranked_topk_counts_with_unit_weight() { vec![topk(5)], None, ); - query(&mut memo, ranked); + query(&mut memo, ranked, None); let memo = pass1_memo(memo, None).unwrap(); let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); @@ -969,7 +970,7 @@ fn ranked_item_is_the_keys_outside_the_partition() { vec![topk(5)], None, ); - query(&mut memo, ranked); + query(&mut memo, ranked, None); let memo = pass1_memo(memo, None).unwrap(); let (_, update) = state_group(&memo, SketchAlgorithm::CountSketchWithHeap); assert_eq!( @@ -1008,7 +1009,7 @@ fn nested_distributive_aggregate_merges_into_the_outer_state_group() { vec![intent], None, ); - query(&mut memo, by_job); + query(&mut memo, by_job, None); let memo = pass1_memo(memo, None).unwrap(); let outer_state = groups_where( &memo, @@ -1046,7 +1047,7 @@ fn nested_merge_needs_the_same_distributive_aggregate() { vec![outer], None, ); - query(&mut memo, by_job); + query(&mut memo, by_job, None); let memo = pass1_memo(memo, None).unwrap(); assert!(memo .groups() @@ -1071,14 +1072,14 @@ fn update_weights_follow_the_measure() { vec![AggIntent::Count], None, ); - query(&mut memo, counts); + query(&mut memo, counts, None); let memo = pass1_memo(memo, None).unwrap(); let (_, update) = state_group(&memo, SketchAlgorithm::Cms); assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); let mut memo = Memo::new(&DataWorkload::default()); let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); let sums = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); - query(&mut memo, sums); + query(&mut memo, sums, None); let memo = pass1_memo(memo, None).unwrap(); let state = groups_where( &memo, @@ -1189,7 +1190,7 @@ fn a_quantile_outside_the_unit_interval_offers_no_summary() { #[test] fn keyed_heap_input_refuses_a_filtered_count() { let mut memo = Memo::new(&DataWorkload::default()); - let rows = scan_group(&mut memo, "m", &["job"]); + let rows = scan_group(&mut memo, "m", &["job"], vec![]); let mut count_by_job = |filter: Option>| { group( &mut memo, @@ -1321,8 +1322,8 @@ fn assert_unsupported_way_adds_nothing( #[test] fn a_heap_estimate_without_the_groups_columns_adds_nothing() { let mut memo = Memo::new(&DataWorkload::default()); - let by_job = scan_group(&mut memo, "m", &["job"]); - let by_host = scan_group(&mut memo, "n", &["host"]); + let by_job = scan_group(&mut memo, "m", &["job"], vec![]); + let by_host = scan_group(&mut memo, "n", &["host"], vec![]); let ranked = aggregate_group( &mut memo, by_job, diff --git a/crates/stage1-pass2-reuse-candidates/Cargo.toml b/crates/stage1-pass2-reuse-candidates/Cargo.toml index 430c48f0f..88deb2865 100644 --- a/crates/stage1-pass2-reuse-candidates/Cargo.toml +++ b/crates/stage1-pass2-reuse-candidates/Cargo.toml @@ -3,12 +3,16 @@ name = "asap-stage1-pass2-reuse-candidates" version = "0.1.0" edition = "2021" -# #509 Stage 1, Pass 2: the reuse rules over memo groups and the logical +# Stage 1, Pass 2: the reuse rules over memo groups and the logical # edges (`asap_types::stages::LogicalASAPMemoCandidates`) Stage 2 expands. # Reads Pass 1's memo through `asap_types::stages`; depends on no other # stage crate. [dependencies] asap-types = { path = "../types" } -# The deployment's measured accuracy (`AccuracyEvidence`) Pass 2 reads. +# The accuracy rules Pass 2 applies: readout levels, the metric rule, +# alpha^log and the deployment's measured accuracy (`AccuracyEvidence`). asap-accuracy = { path = "../accuracy" } thiserror = "2" + +[dev-dependencies] +asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs new file mode 100644 index 000000000..66e15d87b --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs @@ -0,0 +1,192 @@ +//! Step 4, dead-candidate elimination (concepts doc, section 3): a +//! candidate that is not a query output is dead when it never had a +//! consumer, or when every edge to a consumer fails or reaches a dead +//! consumer; any candidate is dead when every edge on one of its inputs +//! fails or reads a dead producer. Elimination repeats until nothing +//! changes. Each dead candidate is listed with every root cause, one +//! shortest path per failing edge. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +use crate::edges::CandidateEdge; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, PruneCause, PrunedLogicalCandidate, RootPruneReason, +}; + +/// The dead candidates of `memo`, whose possible edges are `edges`, each +/// with its causes, in candidate order. +pub(crate) fn eliminate_dead_candidates( + memo: &Memo, + edges: &[CandidateEdge], +) -> Vec { + let graph = EdgeGraph::new(edges); + let outputs = memo.root_groups(); + let all: Vec = memo + .groups() + .iter() + .enumerate() + .flat_map(|(group, memo_group)| { + (0..memo_group.candidates.len()).map(move |index_in_group| CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }) + }) + .collect(); + // Per dead candidate, why, judged against those dead before it: cut + // off from an input (`Some(input)`), or without a live consumer. + let mut dead: BTreeMap> = BTreeMap::new(); + loop { + let newly_dead: Vec<(CandidateForMemoGroupId, Option)> = all + .iter() + .copied() + .filter(|candidate| !dead.contains_key(candidate)) + .filter_map(|candidate| { + if let Some(input) = graph.cut_off_input(memo, candidate, &dead) { + Some((candidate, Some(input))) + } else if !outputs.contains(&candidate.group) + && !graph.has_live_consumer(candidate, &dead) + { + Some((candidate, None)) + } else { + None + } + }) + .collect(); + if newly_dead.is_empty() { + break; + } + dead.extend(newly_dead); + } + dead.keys() + .map(|&candidate| PrunedLogicalCandidate { + candidate, + causes: graph.causes(candidate, &dead), + }) + .collect() +} + +/// `edges` by producer and by (consumer, input). +struct EdgeGraph<'a> { + by_producer: HashMap>, + by_consumer_input: HashMap<(CandidateForMemoGroupId, usize), Vec<&'a CandidateEdge>>, +} + +impl<'a> EdgeGraph<'a> { + fn new(edges: &'a [CandidateEdge]) -> Self { + let mut graph = Self { + by_producer: HashMap::new(), + by_consumer_input: HashMap::new(), + }; + for edge in edges { + graph + .by_producer + .entry(edge.producer) + .or_default() + .push(edge); + graph + .by_consumer_input + .entry((edge.consumer, edge.input)) + .or_default() + .push(edge); + } + graph + } + + /// Whether `candidate` has a satisfied edge into a consumer not in `dead`. + fn has_live_consumer( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> bool { + self.by_producer.get(&candidate).is_some_and(|edges| { + edges + .iter() + .any(|edge| edge.satisfies.is_ok() && !dead.contains_key(&edge.consumer)) + }) + } + + /// The first input of `candidate` with no satisfied edge from a producer + /// not in `dead`: every edge on it is cut (the metric rule, a missed + /// requirement) or reads an eliminated producer. + fn cut_off_input( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Option { + (0..memo + .candidate(candidate) + .expression + .operator + .children() + .len()) + .find(|&input| { + !self + .by_consumer_input + .get(&(candidate, input)) + .is_some_and(|edges| { + edges.iter().any(|edge| { + edge.satisfies.is_ok() && !dead.contains_key(&edge.producer) + }) + }) + }) + } + + /// Every root cause of dead `candidate`, breadth first, so each failing + /// edge is reached by a shortest path: a candidate cut off from an + /// input (`dead`) is traced along that input's edges, through + /// eliminated producers; any other through its consumer edges and + /// eliminated consumers, ending in `NoConsumer` at one that never had a + /// consumer. + fn causes( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Vec { + let mut causes = Vec::new(); + let mut visited = BTreeSet::from([candidate]); + let mut queue = VecDeque::from([(candidate, Vec::new())]); + while let Some((at, path)) = queue.pop_front() { + // A candidate cut off from an input is traced to its producers, + // any other to its consumers. + let edges = match dead[&at] { + Some(input) => self + .by_consumer_input + .get(&(at, input)) + .map_or(&[][..], Vec::as_slice), + None => match self.by_producer.get(&at) { + Some(edges) => edges.as_slice(), + None => { + causes.push(PruneCause { + through_pruned_candidates: path, + root: RootPruneReason::NoConsumer, + }); + continue; + } + }, + }; + for edge in edges { + match &edge.satisfies { + Err(reason) => causes.push(PruneCause { + through_pruned_candidates: path.clone(), + root: reason.clone(), + }), + // A satisfied edge of a dead candidate leads to a dead + // consumer or producer. + Ok(()) => { + let next = match dead[&at] { + Some(_) => edge.producer, + None => edge.consumer, + }; + if visited.insert(next) { + let mut further = path.clone(); + further.push(next); + queue.push_back((next, further)); + } + } + } + } + } + causes + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/edges.rs b/crates/stage1-pass2-reuse-candidates/src/edges.rs new file mode 100644 index 000000000..9e8073512 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/edges.rs @@ -0,0 +1,114 @@ +//! Step 6, the logical edges: every possible $(u, c, i)$ of the memo with +//! $\alpha^{\log}$, which dead-candidate elimination reads and whose +//! satisfied edges between survivors are Pass 2's output. + +use std::collections::BTreeMap; + +use asap_accuracy::{ + metrics_accepted_on_inputs, producer_satisfies_logical_requirements, + readout_metric_of_candidate, AccuracyEvidence, +}; +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroupId, EvaluationGrid, Memo, MemoGroupId, RootPruneReason, +}; + +/// One possible $(u, c, i)$: `producer` is a candidate of the group input +/// `input` of `consumer` reads, and whether it satisfies that input's +/// logical requirement (`producer_satisfies_logical_requirements`). +pub(crate) struct CandidateEdge { + pub(crate) producer: CandidateForMemoGroupId, + pub(crate) consumer: CandidateForMemoGroupId, + pub(crate) input: usize, + pub(crate) satisfies: Result<(), RootPruneReason>, +} + +/// Every $(u, c, i)$ of `memo` (`Memo::producers`), grouped by consumer and +/// input, producers in group order; `accepted` is +/// `asap_accuracy::accepted_metrics` of `memo`. An edge into a readout of +/// `readouts_without_per_value_failure_probability` fails with +/// `ValuesPerEvaluationUnknown`. A chain `SummaryTumblingWindowChain[w]` +/// read by a consumer group whose result depends on the evaluation time +/// fails its edges with `ChainWidthDoesNotDivideEvaluationGrid` unless $w$ +/// divides that group's evaluation grid (`evaluation_grid_of_group`, from +/// `evaluation_grids`): a chain of another reader's width does not end at +/// this reader's evaluations. +pub(crate) fn candidate_edges( + memo: &Memo, + accepted: &[Option>], + evaluation_grid_of_group: &[Option], + readouts_without_per_value_failure_probability: &BTreeMap, + evidence: &dyn AccuracyEvidence, +) -> Vec { + let readout_metrics = readout_metric_of_candidate(memo); + let mut edges = Vec::new(); + for (group, memo_group) in memo.groups().iter().enumerate() { + // The width of a chain input whose windows do not end at this + // group's evaluation times, and the group's evaluation period. + let chain_off_the_evaluation_grid = + |input_group: MemoGroupId| match memo.groups()[input_group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) if memo_group.group_properties.depends_on_evaluation_time => { + let evaluation_period_ms = + evaluation_grid_of_group[group].map(|grid| grid.period_ms); + (!evaluation_period_ms.is_some_and(|period_ms| { + period_ms.is_multiple_of(tumbling_window_width_ms) + })) + .then_some((tumbling_window_width_ms, evaluation_period_ms)) + } + _ => None, + }; + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let consumer = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + let metrics_accepted = metrics_accepted_on_inputs(memo, accepted, consumer); + for (input, &input_group) in candidate + .expression + .operator + .children() + .into_iter() + .enumerate() + { + let chain_off_the_grid = chain_off_the_evaluation_grid(input_group); + for producer in memo.producers(consumer, input) { + edges.push(CandidateEdge { + producer, + consumer, + input, + satisfies: match ( + chain_off_the_grid, + readouts_without_per_value_failure_probability.get(&consumer), + ) { + (Some((tumbling_window_width_ms, evaluation_period_ms)), _) => { + Err(RootPruneReason::ChainWidthDoesNotDivideEvaluationGrid { + consumer, + input, + tumbling_window_width_ms, + evaluation_period_ms, + }) + } + (None, Some(&query)) => { + Err(RootPruneReason::ValuesPerEvaluationUnknown { consumer, query }) + } + (None, None) => producer_satisfies_logical_requirements( + memo, + &readout_metrics, + metrics_accepted.as_deref(), + producer, + consumer, + input, + evidence, + ), + }, + }); + } + } + } + } + edges +} diff --git a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs new file mode 100644 index 000000000..e6d40776f --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs @@ -0,0 +1,109 @@ +//! The evaluation grid of each group, per query reaching it: the times a +//! group's result is evaluated at, which a chain's width must divide. + +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; + +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{EvaluationGrid, Memo}; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; + +/// Per group, by group id: per query reaching it, the period in ms of the +/// grid the query evaluates the group's result on (phase 0), `None` when +/// that grid is unknown. Walking down from each query's root, which is on +/// the query's own grid ([`query_evaluation_grid_ms`]), a group is on its +/// consumer's grid, except below a PromQL subquery: `[10m:30s]` +/// evaluates `` at the multiples of 30 s whatever the outer grid, and +/// `[10m:]` at the server's default interval, unknown to the planner. +/// A group a query reaches through several paths is on the gcd of their +/// grids (unknown when one is). Every candidate's inputs are walked, so a +/// rollup's finer group is on the grids of the queries rolled up. +pub(crate) fn evaluation_grid_per_query( + memo: &Memo, + entries: &[QueryWorkloadEntry], +) -> Vec>> { + let mut grids: Vec>> = vec![BTreeMap::new(); memo.groups().len()]; + let mut pending = Vec::new(); + for (query, memo_query) in memo.queries().iter().enumerate() { + let grid = query_evaluation_grid_ms(entries, query); + for &root in memo_query.root.operator_refs() { + if merge(&mut grids[root.0 as usize], query, grid) { + pending.push(root); + } + } + } + while let Some(group) = pending.pop() { + let grids_here = grids[group.0 as usize].clone(); + for candidate in &memo.groups()[group.0 as usize].candidates { + let operator = &candidate.expression.operator; + let subquery_resolution_ms = match operator { + Operator::NonASAP(NonASAPOp::PromqlSubquery { resolution, .. }) => Some( + resolution + .map(|resolution| resolution.as_millis() as u64) + .filter(|&ms| ms > 0), + ), + _ => None, + }; + for &child in operator.children() { + for (&query, &grid) in &grids_here { + let below = subquery_resolution_ms.unwrap_or(grid); + if merge(&mut grids[child.0 as usize], query, below) { + pending.push(child); + } + } + } + } + } + grids +} + +/// Per group: the grid its result is evaluated on, the gcd of the +/// evaluation grids of the queries reaching it +/// ([`evaluation_grid_per_query`]) at phase 0; `None` when no query +/// reaches it or one evaluates it on no known grid. +pub(crate) fn evaluation_grids( + memo: &Memo, + entries: &[QueryWorkloadEntry], +) -> Vec> { + evaluation_grid_per_query(memo, entries) + .into_iter() + .map(|grids| { + let mut period_ms = 0; + for grid in grids.into_values() { + period_ms = greatest_common_divisor(period_ms, grid?); + } + (period_ms > 0).then_some(EvaluationGrid { + period_ms, + phase_ms: 0, + }) + }) + .collect() +} + +/// The evaluation grid of query `query` (`entries[query]`'s +/// `QueryRecurrence::evaluation_grid_ms`); `None` when it has no known +/// grid. +fn query_evaluation_grid_ms(entries: &[QueryWorkloadEntry], query: usize) -> Option { + let entry = entries.get(query)?; + entry.recurrence.evaluation_grid_ms(entry.range_evaluation) +} + +/// Adds `query`'s `grid` to a group's grids: the gcd with a known one, +/// unknown when either is; whether they changed. +fn merge(grids: &mut BTreeMap>, query: usize, grid: Option) -> bool { + match grids.entry(query) { + Entry::Vacant(entry) => { + entry.insert(grid); + true + } + Entry::Occupied(mut entry) => { + let merged = match (*entry.get(), grid) { + (Some(known), Some(grid)) => Some(greatest_common_divisor(known, grid)), + _ => None, + }; + let changed = merged != *entry.get(); + entry.insert(merged); + changed + } + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/exact_only_groups.rs b/crates/stage1-pass2-reuse-candidates/src/exact_only_groups.rs new file mode 100644 index 000000000..5c682f45b --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/exact_only_groups.rs @@ -0,0 +1,90 @@ +//! The groups each query may read only exactly +//! (`LogicalASAPMemoCandidates::exact_only_groups_of_query`): the metric +//! rule per query, where dead-candidate elimination applies it per group. + +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet}; + +use asap_accuracy::metrics_carried; +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoGroupId}; + +/// Per query: the groups below its root where an error in no metric of its +/// target reaches the root bounded, with every group below them. Walking +/// down from each root with a target, which accepts the target's metric +/// (`AccuracyTarget::metrics`), a candidate passes to its inputs the +/// metrics its error contract carries (`metrics_carried`); a group +/// keeps, per query, the metrics every path to it carries, so one path +/// through a consumer without a contract (a comparison, a sum) makes it +/// exact-only for that query. `read_only_exactly` adds, per query, groups +/// listed by the readout requirements (a readout whose level another query +/// gave, for a query whose failure probability has no per-value one). +/// `order` is the memo's topological order (`Memo::topological_order`). +pub(crate) fn exact_only_groups_of_query( + memo: &Memo, + order: &[MemoGroupId], + read_only_exactly: &BTreeMap>, +) -> Vec> { + let mut accepted: Vec>> = + vec![BTreeMap::new(); memo.groups().len()]; + for (query, memo_query) in memo.queries().iter().enumerate() { + let (Some(target), QueryRoot::Operator(root)) = + (&memo_query.accuracy_target, &memo_query.root) + else { + continue; + }; + accepted[root.0 as usize].insert(query, target.metrics()); + } + let mut exact_only = vec![BTreeSet::new(); memo.queries().len()]; + for (&query, groups) in read_only_exactly { + exact_only[query].extend(groups.iter().copied()); + } + for &group in order.iter().rev() { + let accepted_here = accepted[group.0 as usize].clone(); + for (&query, metrics) in &accepted_here { + if metrics.is_empty() { + exact_only[query].insert(group); + } + } + for candidate in &memo.groups()[group.0 as usize].candidates { + let operator = &candidate.expression.operator; + let carried = metrics_carried(memo, operator, &AccuracyMetric::ALL); + for &child in operator.children() { + for (&query, metrics) in &accepted_here { + let passed: Vec = metrics + .iter() + .copied() + .filter(|metric| carried.contains(metric)) + .collect(); + match accepted[child.0 as usize].entry(query) { + Entry::Vacant(entry) => { + entry.insert(passed); + } + Entry::Occupied(mut entry) => { + entry.get_mut().retain(|metric| passed.contains(metric)) + } + } + } + } + } + } + // Every group below an exact-only group is read exactly too. + for &group in order.iter().rev() { + for groups in &mut exact_only { + if groups.contains(&group) { + for candidate in &memo.groups()[group.0 as usize].candidates { + groups.extend( + candidate + .expression + .operator + .children() + .into_iter() + .copied(), + ); + } + } + } + } + exact_only +} diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs index db7860446..498327bde 100644 --- a/crates/stage1-pass2-reuse-candidates/src/lib.rs +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -1,4 +1,4 @@ -//! `asap-stage1-pass2-reuse-candidates` — #509 Stage 1, Pass 2: the reuse +//! `asap-stage1-pass2-reuse-candidates` — Stage 1, Pass 2: the reuse //! rules over memo groups, the readouts' required accuracy, dead-candidate //! elimination and the logical edges. Its entry, //! [`generate_logical_asap_memo_candidates`], extends Pass 1's memo into @@ -7,25 +7,66 @@ //! count reading one state. //! //! It depends on `asap-types`, where its input and output are, and on -//! `asap-accuracy` for $\alpha^{\log}$. It depends on no other stage crate, -//! and the planner facade calls it after Pass 1. +//! `asap-accuracy` for the readout levels and $\alpha^{\log}$. It depends on +//! no other stage crate, and the planner facade calls it after Pass 1. +/// Step 4: dead-candidate elimination and its root causes. +mod dead_candidates; +/// Step 6: the possible edges and their $\alpha^{\log}$. +mod edges; +/// The evaluation grid of each group, per query reaching it. +mod evaluation_grids; +/// The groups each query may read only exactly. +mod exact_only_groups; +/// Step 3: readout requirements. +mod readout_requirements; /// Steps 1, 2 and 5: the rules that add groups and candidates. mod rules; +#[cfg(test)] +mod test_fixtures; +#[cfg(test)] +mod tests; -use asap_accuracy::AccuracyEvidence; -use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; +use std::collections::BTreeSet; + +use asap_accuracy::{accepted_metrics, AccuracyEvidence}; +use asap_types::stages::{ + CandidateError, CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalEdge, Memo, + MemoGroupError, PrunedLogicalCandidate, +}; use asap_types::workload::QueryWorkloadEntry; -pub use rules::Pass2RuleError; -use rules::{rollup, topk}; +use dead_candidates::eliminate_dead_candidates; +use edges::candidate_edges; +use evaluation_grids::evaluation_grids; +use exact_only_groups::exact_only_groups_of_query; +use readout_requirements::set_readout_requirements; +use rules::{chains, rollup, topk}; + +/// Why Pass 2 failed on a memo: a rule built an expression the node rules +/// reject (a bug in the rule), or the memo's reads form a cycle. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + /// A group, or one of its candidates, reads itself through other groups. + #[error("memo dependencies contain a cycle")] + CyclicDependencies, + /// A rule built a group expression the node rules reject. + #[error("a rule built a group that breaks the node rules: {0}")] + Group(#[from] MemoGroupError), + /// A rule built a candidate expression the node rules reject, or gave + /// it θ its group's output cannot have. + #[error("a rule built a candidate that breaks the node rules: {0}")] + Candidate(CandidateError), +} /// Stage 1, Pass 2 (concepts doc, section 3, "Stage 1, Pass 2"), in order: /// 1. rollup from the finest common group; /// 2. top-k reuse; -/// 3. reach, caps and readout requirements; +/// 3. reach, caps and readout requirements, and the groups each query may +/// read only exactly; /// 4. dead-candidate elimination; /// 5. chain groups for the surviving state groups, then dead-candidate -/// elimination again over the memo with them; +/// elimination again over the memo with them (a merge whose θ serves no +/// consumer); /// 6. the logical edges between the surviving candidates, with /// $\alpha^{\log}$ from `evidence` or the theoretical guarantee. /// @@ -38,9 +79,54 @@ pub fn generate_logical_asap_memo_candidates( let mut memo = pass1; rollup::add_rollup_candidates(&mut memo)?; topk::add_topk_candidates(&mut memo)?; - let _ = (memo, entries, evidence); - todo!( - "#1207 PR 4: steps 3-6: readout requirements, dead-candidate elimination, \ - chain groups (`rules::chains::add_chain_groups`) and the logical edges" - ) + let topological_order = |memo: &Memo| { + memo.topological_order() + .ok_or(Pass2RuleError::CyclicDependencies) + }; + let order = topological_order(&memo)?; + let (readouts_without_per_value_failure_probability, read_only_exactly) = + set_readout_requirements(&mut memo, &order)?; + let candidates_of = |pruned: &[PrunedLogicalCandidate]| -> BTreeSet { + pruned.iter().map(|pruned| pruned.candidate).collect() + }; + let before_chains = eliminate_dead_candidates( + &memo, + &candidate_edges( + &memo, + &accepted_metrics(&memo, &order), + &evaluation_grids(&memo, entries), + &readouts_without_per_value_failure_probability, + evidence, + ), + ); + chains::add_chain_groups(&mut memo, entries, &candidates_of(&before_chains))?; + let order = topological_order(&memo)?; + let evaluation_grid_of_group = evaluation_grids(&memo, entries); + let candidate_edges = candidate_edges( + &memo, + &accepted_metrics(&memo, &order), + &evaluation_grid_of_group, + &readouts_without_per_value_failure_probability, + evidence, + ); + let pruned = eliminate_dead_candidates(&memo, &candidate_edges); + let dead = candidates_of(&pruned); + let edges = candidate_edges + .into_iter() + .filter(|edge| !dead.contains(&edge.producer) && !dead.contains(&edge.consumer)) + .map(|edge| LogicalEdge { + producer_candidate_id: edge.producer, + consumer_candidate_id: edge.consumer, + input_index_of_consumer: edge.input, + satisfies_logical_requirement: edge.satisfies.is_ok(), + }) + .collect(); + let exact_only_groups_of_query = exact_only_groups_of_query(&memo, &order, &read_only_exactly); + Ok(LogicalASAPMemoCandidates { + memo, + edges, + pruned, + evaluation_grid_of_group, + exact_only_groups_of_query, + }) } diff --git a/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs new file mode 100644 index 000000000..1547b574f --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/readout_requirements.rs @@ -0,0 +1,86 @@ +//! Step 3 (concepts doc, section 3, "Reach, caps and readout requirements"; +//! section 4, "Readout requirements"): writes each readout's required +//! accuracy, from the levels `asap_accuracy::readout_levels` gives it, on +//! the readout and its copies. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::Pass2RuleError; +use asap_accuracy::readout_levels; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, RequiredAccuracy, +}; + +/// Gives every sketch readout (`SummaryEstimate` over a sketch state) its +/// required accuracy, one readout candidate per requirement level +/// ([`readout_levels`]): Pass 1's candidate takes the first level and a +/// copy is added per further level, each written with +/// `Memo::set_candidate_input_requirements`. A readout without a level +/// keeps no requirement. `order` is the memo's topological order. +/// +/// Returns, first, the readouts that no query gives a level while one +/// reaching them has a failure probability without a per-value one, each +/// with that query: Pass 2 prunes them +/// (`RootPruneReason::ValuesPerEvaluationUnknown`); second, per such query, +/// the readout groups it reaches without a per-value failure probability, +/// which it may read only exactly (another query's level says nothing of +/// its own δ). +#[allow(clippy::type_complexity)] +pub(crate) fn set_readout_requirements( + memo: &mut Memo, + order: &[MemoGroupId], +) -> Result< + ( + BTreeMap, + BTreeMap>, + ), + Pass2RuleError, +> { + let mut readouts_without_per_value_failure_probability = BTreeMap::new(); + let mut read_only_exactly: BTreeMap> = BTreeMap::new(); + for (readout, (levels, queries_without_per_value_failure_probability)) in + readout_levels(memo, order) + { + for &query in &queries_without_per_value_failure_probability { + read_only_exactly + .entry(query) + .or_default() + .insert(readout.group); + } + match queries_without_per_value_failure_probability.first() { + Some(&query) if levels.is_empty() => { + readouts_without_per_value_failure_probability.insert(readout, query); + } + _ => write_levels(memo, readout, &levels)?, + } + } + Ok(( + readouts_without_per_value_failure_probability, + read_only_exactly, + )) +} +/// Writes `levels` on `readout`: the first on it, each further one on a +/// copy of it added to its group. No level leaves it without a +/// requirement. +fn write_levels( + memo: &mut Memo, + readout: CandidateForMemoGroupId, + levels: &[RequiredAccuracy], +) -> Result<(), Pass2RuleError> { + let unset = memo.candidate(readout).clone(); + for (n, &level) in levels.iter().enumerate() { + let id = if n == 0 { + readout + } else { + memo.add_candidate(readout.group, unset.clone()) + .map_err(Pass2RuleError::Candidate)? + }; + memo.set_candidate_input_requirements( + id, + vec![LogicalRequirements { + required_accuracy: Some(level), + }], + ); + } + Ok(()) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules.rs b/crates/stage1-pass2-reuse-candidates/src/rules.rs index 039b01b89..ff7857b8f 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules.rs @@ -1,17 +1,84 @@ //! Pass 2's rules over groups (concepts doc, section 3, "Stage 1, Pass 2"): //! each reads the memo and adds groups (`Memo::find_or_add_group`) or -//! candidates (`Memo::add_candidate`); none copies a consumer. +//! candidates (`Memo::add_candidate`); none copies a consumer, so every +//! consumer of a group sees what a rule adds to it. pub(crate) mod chains; pub(crate) mod rollup; pub(crate) mod topk; -/// Why a Pass 2 rule failed on a memo. -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, +use crate::Pass2RuleError; +use asap_types::ir::Operator; +use asap_types::stages::{ + CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::SketchParams; + +/// Adds the candidate `operator` with $\boldsymbol\theta$ `parameters` to +/// `group`, its inputs without a requirement (step 3 sets the readouts'). +/// `None` when its output is not the group's +/// (`CandidateError::OutputDiffersFromGroup`): the rules match shapes and +/// let the memo decide whether a match computes the group's result. Any +/// other rejection is a bug in the rule and an error. +pub(crate) fn add_candidate_if_output_matches( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, + origin: OriginOfCandidateForMemoGroup, +) -> Result, Pass2RuleError> { + match memo.add_candidate( + group, + CandidateForMemoGroup::new(operator, parameters, origin, Vec::new()), + ) { + Ok(id) => Ok(Some(id)), + Err(CandidateError::OutputDiffersFromGroup { .. }) => Ok(None), + Err(error) => Err(Pass2RuleError::Candidate(error)), + } +} + +/// The distinct θ of `candidates`, in order. +pub(crate) fn distinct_parameters<'a>( + candidates: impl IntoIterator, +) -> Vec> { + let mut parameters = Vec::new(); + for candidate in candidates { + let theta = candidate.logical_properties.parameters().cloned(); + if !parameters.contains(&theta) { + parameters.push(theta); + } + } + parameters +} + +/// Adds `operator`, which passes on the states or rows of its one input +/// group (a `SummaryMerge`, or a `Filter`, `TimeRange` or `TimeShift`), to +/// `group`: one candidate per θ among the input group's candidates (concepts +/// doc, section 2: such a candidate carries the θ of the states it reads), +/// one without a θ over rows or exact states. +pub(crate) fn add_candidates_per_input_parameters( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result<(), Pass2RuleError> { + let input = *operator.children()[0]; + for theta in distinct_parameters(&memo.groups()[input.0 as usize].candidates) { + add_candidate_if_output_matches(memo, group, operator.clone(), theta, origin.clone())?; + } + Ok(()) +} + +/// The group of `operator`, with its own expression as its candidates +/// ([`add_candidates_per_input_parameters`]): a group a rule adds between +/// others, a compensating `Filter`, a `TimeRange` or `TimeShift` over a +/// chain. +pub(crate) fn add_group_computing_itself( + memo: &mut Memo, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result { + let group = memo.find_or_add_group(operator.clone())?; + add_candidates_per_input_parameters(memo, group, operator, origin)?; + Ok(group) } diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs index 3b64862f2..f03dcde82 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs @@ -5,22 +5,255 @@ use std::collections::BTreeSet; -use crate::rules::Pass2RuleError; -use asap_types::stages::{CandidateForMemoGroupId, Memo}; -use asap_types::workload::QueryWorkloadEntry; - -/// For each summary state group with a build that survived dead-candidate -/// elimination (not in `pruned`), a chain group and a merge candidate per -/// tumbling width $w$ from the gcd closure of the evaluation grids, ranges -/// and offsets of the queries that reach it and repeat on a known grid. +use crate::evaluation_grids::evaluation_grid_per_query; +use crate::rules::{ + add_candidate_if_output_matches, add_candidates_per_input_parameters, + add_group_computing_itself, distinct_parameters, +}; +use crate::Pass2RuleError; +use asap_types::ir::operator::arguments::{AtModifier, TimeShift}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::SummaryTypeCapability; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; + +/// For each summary state group S = `SummaryAgg(TimeRange(W, +/// TimeShift*(X)))` of a summary type that merges, with a build that +/// survived dead-candidate elimination (`pruned`), and each width $w$: the +/// chain group $G_w$ = `SummaryTumblingWindowChain[w](X)` (the shifts +/// stripped, so ranges and offsets over X share it) with one build per θ of +/// S's surviving builds, the shifts and the `TimeRange` re-wrapped around +/// it, and the candidates `SummaryMerge` over the outermost of them in S, +/// each of these one per θ of the group it reads (a chain shared by several +/// state groups has the builds of all of them). +/// +/// The widths are the gcd closure of $\gcd(g_q, W, o_1, \ldots)$ over the +/// queries reaching S whose evaluation grid at S, $g_q$, is known +/// ([`evaluation_grid_per_query`]: below a subquery, its resolution), with +/// the offsets $o_j$ on S's +/// path; no query with a grid, no chain. The edges from a chain whose +/// width does not divide the evaluation grid of the group reading it (the +/// gcd over all its readers) fail with +/// `ChainWidthDoesNotDivideEvaluationGrid` (`candidate_edges`), so the +/// merge over it is pruned: a 15 s chain never serves a reader every 10 s. +/// Under `@ T` S adds no width: it +/// reads only chains other states created whose width divides W, the +/// offsets and T. Under `@ end()` or `@ start()` (a `TimeShift` the front +/// end puts above the subtree it anchors) no chain is read. /// `entries[q]` is query `q`'s workload entry. -// Step 5 runs after dead-candidate elimination, which #1207 PR 4 adds. -#[allow(dead_code)] pub(crate) fn add_chain_groups( memo: &mut Memo, entries: &[QueryWorkloadEntry], pruned: &BTreeSet, ) -> Result<(), Pass2RuleError> { - let _ = (memo, entries, pruned); - todo!("#1207 PR 4: step 5, chain groups and the merges over them") + // First every chain group with its builds, then the groups reading + // them, so a chain shared by state groups offers each every θ it has. + let mut chains_of_states: Vec<(MemoGroupId, ChainPath, Vec)> = Vec::new(); + let grids = evaluation_grid_per_query(memo, entries); + for (index, grids_of_state) in grids.iter().enumerate() { + let state = MemoGroupId(index as u32); + let Some(path) = chain_path(memo, state) else { + continue; + }; + let state_expression = &memo.groups()[index].expression; + let parameters = distinct_parameters( + memo.groups()[index] + .candidates + .iter() + .enumerate() + .filter(|&(index_in_group, candidate)| { + let survived = !pruned.contains(&CandidateForMemoGroupId { + group: state, + index_in_group, + }); + survived && candidate.expression.operator == *state_expression + }) + .map(|(_, candidate)| candidate), + ); + if parameters.is_empty() { + continue; + } + let divisor = path.shifts.iter().fold(path.window_ms, |gcd, shift| { + greatest_common_divisor(gcd, shift.offset_ms.unsigned_abs()) + }); + let widths: BTreeSet = match path.anchored_at_ms { + Some(at_ms) => existing_chain_widths(memo, &path) + .into_iter() + .filter(|&width| divisor % width == 0 && at_ms.unsigned_abs() % width == 0) + .collect(), + None => gcd_closure( + grids_of_state + .values() + .filter_map(|&grid| Some(greatest_common_divisor(grid?, divisor))), + ), + }; + let mut chain_groups = Vec::new(); + for width in widths { + let chain = path.chain(width); + let chain_group = memo.find_or_add_group(chain.clone())?; + for theta in ¶meters { + add_candidate_if_output_matches( + memo, + chain_group, + chain.clone(), + theta.clone(), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + chain_groups.push(chain_group); + } + chains_of_states.push((state, path, chain_groups)); + } + for (state, path, chain_groups) in chains_of_states { + for chain_group in chain_groups { + let mut top = chain_group; + for shift in path.shifts.iter().rev() { + top = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: *shift, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + let windows = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_millis(path.window_ms), + kind: TimeRangeKind::Range, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + add_candidates_per_input_parameters( + memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + } + Ok(()) +} + +/// A state group's walk down to the rows its chain reads: +/// `SummaryAgg` → `TimeRange(W)` → `TimeShift`s → X. +struct ChainPath { + /// The `SummaryAgg`, its child to be replaced by X. + build: ASAPOp, + window_ms: u64, + /// Top down. + shifts: Vec, + /// T of an `@ T` among the shifts. + anchored_at_ms: Option, + rows: MemoGroupId, +} + +impl ChainPath { + /// `SummaryTumblingWindowChain[width](X)` with the build's arguments. + fn chain(&self, width: u64) -> Operator { + let ASAPOp::SummaryAgg { + summary_type, + input, + reduction, + filter, + .. + } = &self.build + else { + unreachable!("a chain path starts at a SummaryAgg"); + }; + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: self.rows, + summary_type: summary_type.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: filter.clone(), + tumbling_window_width_ms: width, + }) + } +} + +/// S's [`ChainPath`]: a `SummaryAgg` of a summary type whose states merge +/// (its capability row), over a range `TimeRange(W)` with `W > 0`, over +/// `TimeShift`s with at most one `@ T` and no `@ end()` or `@ start()`. +fn chain_path(memo: &Memo, state: MemoGroupId) -> Option { + let expression = &memo.groups()[state.0 as usize].expression; + let Operator::ASAP( + build @ ASAPOp::SummaryAgg { + child, + summary_type, + .. + }, + ) = expression + else { + return None; + }; + SummaryTypeCapability::of(summary_type).merge_inputs?; + let Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child: below, + }) = &memo.groups()[child.0 as usize].expression + else { + return None; + }; + let window_ms = u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0)?; + let (mut shifts, mut anchored_at_ms, mut rows) = (Vec::new(), None, *below); + while let Operator::NonASAP(NonASAPOp::TimeShift { shift, child }) = + &memo.groups()[rows.0 as usize].expression + { + match shift.at { + None => {} + Some(AtModifier::Timestamp(at_ms)) if anchored_at_ms.is_none() => { + anchored_at_ms = Some(at_ms) + } + Some(_) => return None, + } + shifts.push(*shift); + rows = *child; + } + Some(ChainPath { + build: build.clone(), + window_ms, + shifts, + anchored_at_ms, + rows, + }) +} + +/// The widths of the chain groups over `path`'s rows with its build's +/// arguments that the memo has already. +fn existing_chain_widths(memo: &Memo, path: &ChainPath) -> Vec { + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) if group.expression == path.chain(*tumbling_window_width_ms) => { + Some(*tumbling_window_width_ms) + } + _ => None, + }) + .collect() +} + +/// The gcd closure of `widths`: each, and the gcd of every two in it. +fn gcd_closure(widths: impl IntoIterator) -> BTreeSet { + let mut closure: BTreeSet = BTreeSet::new(); + for width in widths.into_iter().filter(|&width| width > 0) { + let gcds: Vec = closure + .iter() + .map(|&known| greatest_common_divisor(known, width)) + .collect(); + closure.insert(width); + closure.extend(gcds); + } + closure } diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs index b7676185a..7aa931154 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs @@ -3,16 +3,617 @@ //! same definition, and per cluster of equal definitions one finest common //! group $G^*$ every member can roll up from. -use crate::rules::Pass2RuleError; -use asap_types::stages::Memo; +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rules::{ + add_candidate_if_output_matches, add_candidates_per_input_parameters, + add_group_computing_itself, +}; +use crate::Pass2RuleError; +use asap_types::ir::operator::arguments::SourceColumn; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::properties::selection::{constraint_of, Constraint}; +use asap_types::ir::scalar::{CompareOpKind, ExprSemantics, Predicate, ScalarExpr, ScalarValue}; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + listed_build_parameters, AggregateDefinition, ColumnLineage, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::SummaryTypeCapability; /// Adds, to each aggregate group $G_c$, a candidate computing it from each -/// finer group $G_f$ of its cluster, after synthesizing each cluster's -/// finest common group. A cluster is the summary state groups (`SummaryAgg` -/// of a summary type whose states merge across groups) or exact `Aggregate` -/// groups (of sums, counts, minima and maxima) with equal definitions apart -/// from their keys and equal interval constraints (the time window). +/// finer group $G_f$ of its cluster ([`rollup_from`]), after synthesizing +/// each cluster's finest common group ([`finest_common_group`]). A cluster +/// is the summary state groups (`SummaryAgg` of a summary type whose +/// states merge across groups) or exact `Aggregate` groups (of sums, +/// counts, minima and maxima) with equal definitions apart from their keys +/// ([`Member::definition`]) and equal interval constraints (the time +/// window). pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: step 1, rollup from the finest common group") + let mut clusters: Vec> = Vec::new(); + for index in 0..memo.groups().len() { + let Some(member) = Member::of(memo, MemoGroupId(index as u32)) else { + continue; + }; + match clusters + .iter_mut() + .find(|cluster| cluster[0].same_cluster(&member)) + { + Some(cluster) => cluster.push(member), + None => clusters.push(vec![member]), + } + } + for mut cluster in clusters.into_iter().filter(|cluster| cluster.len() > 1) { + if let Some(finest) = finest_common_group(memo, &cluster)? { + if cluster.iter().all(|member| member.group != finest) { + cluster.extend(Member::of(memo, finest)); + } + } + for coarse in &cluster { + for fine in &cluster { + if coarse.group != fine.group { + rollup_from(memo, coarse, fine)?; + } + } + } + } + Ok(()) +} + +/// One aggregate group as a rollup rule reads it. +struct Member { + group: MemoGroupId, + /// The group's expression. + expression: Operator, + /// The rows it aggregates: the `SummaryAgg`'s or `Aggregate`'s input. + rows: MemoGroupId, + /// Its group-by keys, in output order, by the source column they copy. + keys: Vec, + /// Its coverage's selection, one box. + selection: BTreeMap, + /// Its coverage's definition without its keys: the top operator's + /// reduction (and an `Aggregate`'s output names) blanked, and the input + /// written out down to the source ([`Member::see_through_per_series_layers`]), + /// so equal computations over one source compare equal whatever their + /// keys, label selections and the labels their scans list. + definition: AggregateDefinition, +} + +impl Member { + fn of(memo: &Memo, group: MemoGroupId) -> Option { + let memo_group = &memo.groups()[group.0 as usize]; + let (rows, by) = match &memo_group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + reduction: Reduction::Reduce(by), + .. + }) if SummaryTypeCapability::merges_groups(summary_type) => (*child, by), + Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) if filters.iter().all(Option::is_none) + && measures + .iter() + .all(|measure| rolled_up(measure, 0).is_some()) => + { + (*child, by) + } + _ => return None, + }; + if by.is_without() { + return None; + } + let coverage = memo_group.group_properties.aggregate_coverage.as_ref()?; + let [selection] = coverage.selection() else { + return None; + }; + let rows_lineage = &memo.groups()[rows.0 as usize] + .group_properties + .column_lineage; + let keys = by + .keys() + .iter() + .map(|&key| match rows_lineage.get(key)? { + ColumnLineage::SourceColumn(column) => Some(column.clone()), + ColumnLineage::Computed { .. } => None, + }) + .collect::>>()?; + let mut definition = coverage.definition().clone(); + match definition.unlifted_operators.first_mut()? { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => { + *reduction = Reduction::PerEntity + } + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + output_names, + .. + }) => { + *reduction = Reduction::PerEntity; + output_names.clear(); + } + _ => return None, + } + let mut selection = selection.columns().clone(); + Self::see_through_per_series_layers(memo, &mut definition, &mut selection); + Some(Self { + group, + expression: memo_group.expression.clone(), + rows, + keys, + selection, + definition, + }) + } + + /// Writes the input of `definition` out down to its source. Each + /// per-series layer below it ([`per_series_layer`]: a `TimeRange`, a + /// `TimeShift`, a per-series `Aggregate`) becomes an unlifted operator, + /// so two members compare equal only over the same range and the same + /// per-series function. A time-series `Scan` under them is the last + /// unlifted operator: each label matcher on it moves into `selection` + /// (by source column), since a label is constant within a series and a + /// per-series layer keeps or drops whole series alike; the rest stay + /// its predicates. Its schema is dropped when nothing left names a + /// column by position ([`names_columns_by_position`]): the labels a + /// scan lists are those its query reads, its rows (one per sample of + /// each series, the series identity holding every label) are the same + /// whatever it lists, and members name their keys and selections by + /// source column. + fn see_through_per_series_layers( + memo: &Memo, + definition: &mut AggregateDefinition, + selection: &mut BTreeMap, + ) { + while let Some(input) = definition.input { + let expression = &memo.groups()[input.0 as usize].expression; + if !per_series_layer(expression) { + break; + } + definition + .unlifted_operators + .push(expression.map_children(|_| None)); + definition.input = Some(*expression.children()[0]); + } + // The bottom `Scan`: the input group, or the last unlifted operator + // when the coverage lifted one of its predicates already. + let (scan, lineage) = match definition.input { + Some(input) => { + let scan_group = &memo.groups()[input.0 as usize]; + ( + scan_group.expression.map_children(|&child| Some(child)), + Some(&scan_group.group_properties.column_lineage), + ) + } + None => match definition.unlifted_operators.last() { + Some(last) => (last.clone(), None), + None => return, + }, + }; + let Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) = scan + else { + return; + }; + let mut kept = Vec::new(); + for predicate in predicates { + let label_matcher = constraint_of(&predicate.0).and_then(|(column, constraint)| { + let is_label = source.is_time_series() + && schema.fields.get(column)?.dtype == FieldDataType::Plain(DataType::Utf8); + match lineage?.get(column)? { + ColumnLineage::SourceColumn(source_column) + if is_label && !selection.contains_key(source_column) => + { + Some((source_column.clone(), constraint)) + } + _ => None, + } + }); + match label_matcher { + Some((column, constraint)) => { + selection.insert(column, constraint); + } + None => kept.push(predicate), + } + } + if definition.input.take().is_none() { + definition.unlifted_operators.pop(); + } + let schema = if source.is_time_series() + && kept.is_empty() + && !definition + .unlifted_operators + .iter() + .any(names_columns_by_position) + { + Schema::default() + } else { + schema + }; + definition + .unlifted_operators + .push(Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: kept, + schema, + })); + } + + /// Whether `other` computes the same thing over the same time window: + /// equal definitions without keys and equal interval constraints. + fn same_cluster(&self, other: &Self) -> bool { + self.definition == other.definition && self.intervals() == other.intervals() + } + + /// Its interval constraints (the time window, value ranges): equal + /// across a cluster, never compensated. + fn intervals(&self) -> BTreeMap<&SourceColumn, &Constraint> { + self.selection + .iter() + .filter(|(_, constraint)| matches!(constraint, Constraint::Interval { .. })) + .collect() + } +} + +/// The cluster's finest common group $G^*$: its selection is, per column, +/// the union of the members' value-set constraints (none where a member +/// has none); its keys are every member's keys plus each column on which a +/// member selects fewer rows than $G^*$. It is a member when one has that +/// selection and those keys; otherwise it is built over the rows of a +/// member whose selection is the union, with one build per listed θ. +/// `None` when no member's rows have the union selection (a new scan of +/// the union is not built) or a key is not a column of those rows. +fn finest_common_group( + memo: &mut Memo, + cluster: &[Member], +) -> Result, Pass2RuleError> { + let columns: BTreeSet<&SourceColumn> = cluster + .iter() + .flat_map(|member| member.selection.keys()) + .collect(); + let mut hull = BTreeMap::new(); + let mut keys: BTreeSet = cluster + .iter() + .flat_map(|member| member.keys.iter().cloned()) + .collect(); + for column in columns { + let constraints: Option> = cluster + .iter() + .map(|member| member.selection.get(column)) + .collect(); + let union = constraints.and_then(|constraints| { + if constraints + .iter() + .all(|constraint| *constraint == constraints[0]) + { + return Some(constraints[0].clone()); + } + let mut values: Vec = Vec::new(); + for constraint in constraints { + let Constraint::In(more) = constraint else { + return None; + }; + values.extend( + more.iter() + .filter(|value| !values.contains(value)) + .cloned() + .collect::>(), + ); + } + Some(Constraint::In(values)) + }); + if let Some(union) = union { + hull.insert(column.clone(), union); + } + if cluster + .iter() + .any(|member| member.selection.get(column) != hull.get(column)) + { + keys.insert(column.clone()); + } + } + if let Some(member) = cluster.iter().find(|member| { + member.selection == hull && member.keys.iter().cloned().collect::>() == keys + }) { + return Ok(Some(member.group)); + } + let Some(base) = cluster.iter().find(|member| member.selection == hull) else { + return Ok(None); + }; + let rows_lineage = &memo.groups()[base.rows.0 as usize] + .group_properties + .column_lineage; + let mut key_columns = Vec::new(); + for key in &keys { + let Some(column) = column_of(rows_lineage, key) else { + return Ok(None); + }; + key_columns.push(column); + } + key_columns.sort_unstable(); + let mut expression = base.expression.clone(); + match &mut expression { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) + | Operator::NonASAP(NonASAPOp::Aggregate { reduction, .. }) => { + *reduction = Reduction::by(key_columns) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + } + let finest = memo.find_or_add_group(expression.clone())?; + for theta in listed_build_parameters(&expression) { + add_candidate_if_output_matches( + memo, + finest, + expression.clone(), + theta, + OriginOfCandidateForMemoGroup::Rollup, + )?; + } + Ok(Some(finest)) +} + +/// Adds to `coarse` the candidate computing it from `fine` when (concepts +/// doc, section 3, conditions 1–5) its keys are among `fine`'s, its +/// selection lies inside `fine`'s, narrower only on `fine`'s keys, and the +/// cluster's equal definition and time window hold: +/// `Rollup(by coarse's keys)(Filter(compensation, fine))`, without the +/// `Filter` when the selections are equal. States roll up with a +/// regrouping `SummaryMerge`; sums and counts with a sum, minima and maxima +/// with themselves, each under the coarse column's name. The memo checks +/// the result is the coarse group's: a SQL global count (0 over no row) +/// does not roll up from a grouped one (its sum is NULL over no group), +/// while a grouped count does (every group it returns counts a row). +fn rollup_from(memo: &mut Memo, coarse: &Member, fine: &Member) -> Result<(), Pass2RuleError> { + if !coarse.keys.iter().all(|key| fine.keys.contains(key)) { + return Ok(()); + } + let mut compensation = Vec::new(); + for column in coarse.selection.keys().chain(fine.selection.keys()) { + match (coarse.selection.get(column), fine.selection.get(column)) { + (Some(narrow), wide) if Some(narrow) != wide => { + let within = match (narrow, wide) { + (_, None) => true, + (Constraint::In(values), Some(Constraint::In(wider))) => { + values.iter().all(|value| wider.contains(value)) + } + _ => false, + }; + if !within || !fine.keys.contains(column) { + return Ok(()); + } + if !compensation.iter().any(|(known, _)| known == column) { + compensation.push((column.clone(), narrow.clone())); + } + } + (None, Some(_)) => return Ok(()), + _ => {} + } + } + if compensation.is_empty() && coarse.keys.len() == fine.keys.len() { + return Ok(()); + } + let lineage = memo.groups()[fine.group.0 as usize] + .group_properties + .column_lineage + .clone(); + let mut conjuncts = Vec::new(); + for (column, constraint) in &compensation { + let Some(at) = column_of(&lineage, column) else { + return Ok(()); + }; + // The comparison of the frontend that reads the column's source. + let semantics = if column.source.is_time_series() { + ExprSemantics::Promql + } else { + ExprSemantics::Sql + }; + let Some(conjunct) = selecting(at, constraint, semantics) else { + return Ok(()); + }; + conjuncts.push(conjunct); + } + let Some(group_by) = coarse + .keys + .iter() + .map(|key| column_of(&lineage, key)) + .collect::>>() + else { + return Ok(()); + }; + // The compensating `Filter` stays pending, named by the id it would get + // as a new group, until the rollup over it derives the coarse group's + // schema (`Memo::output_schema_of_pending_expression`): a rollup the + // memo would reject adds no group. + let filter = (!conjuncts.is_empty()).then(|| { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(if conjuncts.len() == 1 { + conjuncts.remove(0) + } else { + ScalarExpr::BoolAnd(conjuncts) + }), + child: fine.group, + }) + }); + let mut pending = Vec::new(); + let input = match &filter { + None => fine.group, + Some(filter) => { + let schema = memo.output_schema_of_pending_expression(&[], filter)?; + pending.push((filter.clone(), schema)); + MemoGroupId(u32::try_from(memo.groups().len()).expect("at most u32::MAX groups")) + } + }; + let rollup = match &coarse.expression { + Operator::ASAP(ASAPOp::SummaryAgg { .. }) => Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![input], + group_by: Some(group_by), + }), + Operator::NonASAP(NonASAPOp::Aggregate { + measures, filters, .. + }) => { + let Some(measures) = measures + .iter() + .enumerate() + .map(|(index, measure)| rolled_up(measure, fine.keys.len() + index)) + .collect::>>() + else { + return Ok(()); + }; + // Each rolled-up measure keeps the coarse column's name: a sum + // of counts is named `count`, as the count it computes. + let output_names = memo.groups()[coarse.group.0 as usize] + .group_properties + .schema + .fields + .iter() + .skip(coarse.keys.len()) + .map(|field| field.name.clone()) + .collect(); + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(group_by), + measures, + output_names, + filters: vec![None; filters.len()], + having: None, + child: input, + }) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + }; + if memo + .output_schema_of_pending_expression(&pending, &rollup) + .ok() + .as_ref() + != Some( + &memo.groups()[coarse.group.0 as usize] + .group_properties + .schema, + ) + { + return Ok(()); + } + let rollup = match filter { + None => rollup, + Some(filter) => { + let filter = + add_group_computing_itself(memo, filter, OriginOfCandidateForMemoGroup::Rollup)?; + rollup.map_children(|_| filter) + } + }; + add_candidates_per_input_parameters( + memo, + coarse.group, + rollup, + OriginOfCandidateForMemoGroup::Rollup, + ) +} + +/// The measure rolling `measure`'s finalized values up from column +/// `column` of the finer result: a sum or count by a sum, a minimum or +/// maximum by itself; `None` for any other measure (it rolls up only from +/// states). +fn rolled_up(measure: &AggIntent, column: usize) -> Option { + let col = Some(column); + Some(match measure { + AggIntent::Sum { .. } | AggIntent::Count => AggIntent::Sum { col }, + AggIntent::Min { .. } => AggIntent::Min { col }, + AggIntent::Max { .. } => AggIntent::Max { col }, + _ => return None, + }) +} + +/// Whether `expression` is a per-series layer a rollup sees through: a +/// `TimeRange` or `TimeShift`, or an `Aggregate` computing one value per +/// series (`Reduction::PerEntity`, such as `rate`) without row filters. +fn per_series_layer(expression: &Operator) -> bool { + match expression { + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => true, + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + filters, + having: None, + .. + }) => filters.iter().all(Option::is_none), + _ => false, + } +} + +/// Whether unlifted operator `operator` names a column of its input by +/// position, so it means the same over two scans only when they list the +/// same columns: a measure over a column (`SUM(col)`; a PromQL measure +/// reads the implicit sample value and names none), a row filter, or any +/// operator other than a `SummaryAgg`, an `Aggregate`, a `TimeRange` and a +/// `TimeShift`. The top operator's keys are blanked, and a `SummaryAgg`'s +/// update names its columns by name. +fn names_columns_by_position(operator: &Operator>) -> bool { + match operator { + Operator::ASAP(ASAPOp::SummaryAgg { filter, .. }) => filter.is_some(), + Operator::NonASAP(NonASAPOp::Aggregate { + measures, + filters, + having, + .. + }) => { + having.is_some() + || filters.iter().any(Option::is_some) + || measures + .iter() + .any(|measure| !measure.input_cols().is_empty()) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => false, + _ => true, + } +} + +/// The column of `lineage` that copies `column`, when exactly one does. +fn column_of(lineage: &[ColumnLineage], column: &SourceColumn) -> Option { + let mut at = lineage + .iter() + .enumerate() + .filter(|(_, known)| matches!(known, ColumnLineage::SourceColumn(c) if c == column)) + .map(|(index, _)| index); + let first = at.next()?; + at.next().is_none().then_some(first) +} + +/// The predicate selecting the rows whose column `at` meets `constraint`, +/// with the comparison `semantics` of the frontend reading the column's +/// source: `at = v`, `at != v`, or their disjunction (`IN`) or conjunction +/// (`NOT IN`) over several values. A PromQL `!=` keeps a series without the +/// label (its value is the empty string), which only the frontend's +/// comparison says. `None` for an interval (never a compensation: the time +/// window is equal across a cluster). +fn selecting( + at: usize, + constraint: &Constraint, + semantics: ExprSemantics, +) -> Option> { + let compare = |op: CompareOpKind, value: &ScalarValue| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(at)), + op, + right: Box::new(ScalarExpr::Literal(value.clone())), + semantics, + }; + let (op, values, combine): (_, _, fn(Vec<_>) -> _) = match constraint { + Constraint::In(values) => (CompareOpKind::Eq, values, ScalarExpr::BoolOr), + Constraint::NotIn(values) => (CompareOpKind::Ne, values, ScalarExpr::BoolAnd), + Constraint::Interval { .. } => return None, + }; + Some(match &values[..] { + [value] => compare(op, value), + values => combine( + values + .iter() + .map(|value| compare(op.clone(), value)) + .collect(), + ), + }) } diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs index f90858eff..f6087fd8b 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs @@ -1,14 +1,88 @@ //! Step 2, top-k reuse: an exact top-k reads a larger top-k of the same //! input and partition, whose ranked rows hold its own. -use crate::rules::Pass2RuleError; -use asap_types::stages::Memo; +use crate::rules::add_candidate_if_output_matches; +use crate::Pass2RuleError; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; -/// To each top-k group another top-k group over the same input group and -/// partition with the largest `k` serves, the candidate `TopK(k)` over that -/// group's ranked rows: the `k` highest of the `k_max` highest rows are the -/// `k` highest. Only exact ranked rows give the reuse its answer. +/// To each top-k group (`Aggregate` of one `TopK` measure, partitioned `by` +/// keys, without filters or `HAVING`) another top-k group over the same +/// input group and partition with the largest `k` serves, the candidate +/// `TopK(k)` over that group's ranked rows, partitioned by its leading key +/// columns: the `k` highest of the `k_max` highest rows are the `k` +/// highest. Only exact ranked rows give the reuse its answer: a ranking has +/// no error contract (`asap_accuracy::error_propagation`), so the edge +/// from an approximate producer (a heap's readout) fails the metric rule +/// when a query with an accuracy target reaches it. pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: step 2, top-k reuse") + let topks: Vec<(MemoGroupId, MemoGroupId, Reduction, usize)> = (0..memo.groups().len()) + .filter_map(|index| { + let group = MemoGroupId(index as u32); + let (child, reduction, k) = topk(&memo.groups()[index].expression)?; + Some((group, child, reduction, k)) + }) + .collect(); + for (group, child, reduction, k) in &topks { + let Some((largest, k_max)) = topks + .iter() + .filter(|(_, other_child, other_reduction, _)| { + other_child == child && other_reduction == reduction + }) + .map(|(other, _, _, other_k)| (*other, *other_k)) + .max_by_key(|&(other, other_k)| (other_k, std::cmp::Reverse(other))) + else { + continue; + }; + if k_max <= *k { + continue; + } + let Operator::NonASAP(NonASAPOp::Aggregate { + measures, + output_names, + filters, + .. + }) = memo.groups()[group.0 as usize].expression.clone() + else { + continue; + }; + let keys = reduction.group_keys().map_or(0, |by| by.keys().len()); + add_candidate_if_output_matches( + memo, + *group, + Operator::NonASAP(NonASAPOp::Aggregate { + child: largest, + reduction: Reduction::by((0..keys).collect()), + measures, + output_names, + filters, + having: None, + }), + None, + OriginOfCandidateForMemoGroup::TopKReuse, + )?; + } + Ok(()) +} + +/// The input group, partition and `k` of a top-k group: one `TopK` measure, +/// `by` keys (not `without`), no measure filter, no `HAVING`. +fn topk(operator: &Operator) -> Option<(MemoGroupId, Reduction, usize)> { + let Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: reduction @ Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) = operator + else { + return None; + }; + let [AggIntent::TopK { k, .. }] = measures.as_slice() else { + return None; + }; + (!by.is_without() && filters.iter().all(Option::is_none)) + .then(|| (*child, reduction.clone(), *k)) } diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs new file mode 100644 index 000000000..22911da8a --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -0,0 +1,189 @@ +//! Fixtures this crate's own tests share, beside the memo builders of +//! `asap_types::test_support::memo_builders` (groups with the candidates +//! Pass 1 would add, queries), so no test calls another stage crate: the +//! group builders Pass 2's tests alone use, Pass 2's run and readers of +//! its output. +use std::collections::BTreeSet; + +use asap_accuracy::NoMeasuredGuarantees; +use asap_types::ir::operator::arguments::{AtModifier, Reduction}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroup, CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, + PrunedLogicalCandidate, RequiredAccuracy, +}; +use asap_types::summary_type::ExactKind; +use asap_types::test_support::memo_builders::{group, own_candidate}; +use asap_types::test_support::memo_fixtures; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, + RepetitionInterval, TimestampMs, +}; + +use crate::generate_logical_asap_memo_candidates; + +/// `child[seconds]`. +pub fn range(memo: &mut Memo, child: MemoGroupId, seconds: u64) -> MemoGroupId { + group(memo, memo_fixtures::range(seconds as i64 * 1000, child)) +} + +/// `child offset `, or `child @ at`. +pub fn shift( + memo: &mut Memo, + child: MemoGroupId, + seconds: i64, + at: Option, +) -> MemoGroupId { + group(memo, memo_fixtures::shift(seconds * 1000, at, child)) +} + +/// An exact accumulator state of `kind` over `window` grouped by +/// `reduction`, read by a `FinalizeExactAccumulator` group (the query +/// root): what Pass 1 offers for `sum_over_time` (per series) or for a +/// `sum by (keys)` of `count_over_time` (the count by keys). +pub fn exact_state( + memo: &mut Memo, + window: MemoGroupId, + kind: ExactKind, + reduction: Reduction, +) -> (MemoGroupId, MemoGroupId) { + let state = group( + memo, + memo_fixtures::state_grouped_by(FieldDataType::exact(kind), reduction, window), + ); + let finalized = group( + memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + ); + (state, finalized) +} + +/// A query repeating every `interval_ms` at phase 0 (`FixedIntervalAt`). +pub fn entry(interval_ms: u32) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("q".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + } +} + +pub fn pass2(memo: Memo, entries: &[QueryWorkloadEntry]) -> LogicalASAPMemoCandidates { + generate_logical_asap_memo_candidates(memo, entries, &NoMeasuredGuarantees).unwrap() +} + +pub fn candidates_of(memo: &Memo, group: MemoGroupId) -> &[CandidateForMemoGroup] { + &memo.groups()[group.0 as usize].candidates +} + +/// The edges into input `input` of `consumer`: each producer and α^log. +pub fn edges_into( + logical: &LogicalASAPMemoCandidates, + consumer: CandidateForMemoGroupId, + input: usize, +) -> Vec<(CandidateForMemoGroupId, bool)> { + logical + .edges + .iter() + .filter(|edge| { + edge.consumer_candidate_id == consumer && edge.input_index_of_consumer == input + }) + .map(|edge| { + ( + edge.producer_candidate_id, + edge.satisfies_logical_requirement, + ) + }) + .collect() +} + +/// The pruned entry of `candidate`, if Pass 2 eliminated it. +pub fn pruned_entry( + logical: &LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, +) -> Option<&PrunedLogicalCandidate> { + logical + .pruned + .iter() + .find(|pruned| pruned.candidate == candidate) +} + +/// The required accuracy of each readout candidate of `group` (its only +/// input's). +pub fn requirements_of(memo: &Memo, group: MemoGroupId) -> Vec> { + candidates_of(memo, group) + .iter() + .filter(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }) + .map(|c| c.input_requirements[0].required_accuracy) + .collect() +} + +/// The tumbling-window widths of the chains whose merges state group +/// `state` has as candidates. +pub fn chain_widths_serving(memo: &Memo, state: MemoGroupId) -> BTreeSet { + candidates_of(memo, state) + .iter() + .filter_map(|c| match &c.expression.operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) => chain_width_below(memo, children[0]), + _ => None, + }) + .collect() +} + +/// The width of the tumbling-window chain `group` reads through +/// `TimeRange` and `TimeShift` groups, if any. +pub fn chain_width_below(memo: &Memo, mut group: MemoGroupId) -> Option { + loop { + match &memo.groups()[group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => return Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => group = *child, + _ => return None, + } + } +} + +/// The groups whose expression matches `pattern`. +pub fn groups_where( + memo: &Memo, + pattern: impl Fn(&Operator) -> bool, +) -> Vec { + memo.groups() + .iter() + .enumerate() + .filter(|(_, group)| pattern(&group.expression)) + .map(|(index, _)| MemoGroupId(index as u32)) + .collect() +} + +/// The group whose expression is `operator`, if the memo has one. +pub fn group_of(memo: &Memo, operator: &Operator) -> Option { + groups_where(memo, |expression| expression == operator) + .first() + .copied() +} + +/// The readout of `state` a candidate `id` of a quantile group is or reads. +pub fn readout_of(memo: &Memo, id: CandidateForMemoGroupId) -> CandidateForMemoGroupId { + match &memo.candidate(id).expression.operator { + Operator::NonASAP(NonASAPOp::Project { child, .. }) => own_candidate(*child), + _ => id, + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests.rs b/crates/stage1-pass2-reuse-candidates/src/tests.rs new file mode 100644 index 000000000..68ccba04c --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests.rs @@ -0,0 +1,11 @@ +//! Pass 2's tests, one module per step. Each reads what the rules, the +//! readout requirements, dead-candidate elimination and the edges give for +//! a memo built from `crate::test_fixtures`. +mod chains; +mod dead_candidates; +mod edges; +mod evaluation_grids; +mod exact_only_groups; +mod readout_requirements; +mod rollup; +mod topk; diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs b/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs new file mode 100644 index 000000000..2017984e0 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/chains.rs @@ -0,0 +1,314 @@ +//! Step 5: chain groups. +use std::collections::BTreeSet; +use std::time::Duration; + +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, TimeShift}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, RootPruneReason, +}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; +use asap_types::workload::{ + DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, +}; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// A mergeable state over a range gets one chain group per width with a +/// build per surviving θ; its readout reads the builds and the merges, +/// one per θ. +#[test] +fn a_mergeable_state_reads_a_chain_with_one_build_per_surviving_parameters() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(120_000)]); + let memo = &logical.memo; + assert_eq!(chain_widths_serving(memo, state), BTreeSet::from([60_000])); + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let listed = parameter_list(SketchAlgorithm::Kll).len(); + assert_eq!(candidates_of(memo, chains[0]).len(), listed); + // The builds and one merge per θ. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); +} + +/// No chain for a summary type whose states do not merge (an instant +/// rate), under `@ end()`, or for a query without a known grid. +#[test] +fn no_chain_without_a_merge_an_anchor_or_a_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = range(&mut memo, input, 300); + let (irate, irate_root) = + exact_state(&mut memo, window, ExactKind::IRate, Reduction::PerEntity); + let anchored = shift(&mut memo, input, 0, Some(AtModifier::End)); + let anchored_window = range(&mut memo, anchored, 300); + let (pinned, pinned_root) = exact_state( + &mut memo, + anchored_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + let other = scan_group(&mut memo, "n", &["job"], vec![]); + let other_window = range(&mut memo, other, 300); + let (no_grid, no_grid_root) = exact_state( + &mut memo, + other_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + for root in [irate_root, pinned_root, no_grid_root] { + query(&mut memo, root, None); + } + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(60_000), without_phase]); + for state in [irate, pinned, no_grid] { + assert!( + chain_widths_serving(&logical.memo, state).is_empty(), + "G{}", + state.0 + ); + } + assert!(groups_where(&logical.memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + )) + .is_empty()); +} + +/// A chain serves a reader only when its width divides the reader's +/// evaluation grid: `quantile_over_time(0.9, m[30s])` read every 15 s and +/// every 10 s is one state group evaluated on the 5 s grid, so of its 5, +/// 10 and 15 s chains only the 5 s one has edges, and the merges over the +/// others are pruned (a 15 s chain would answer the 10 s reader at times +/// it does not end on). +#[test] +fn a_chain_whose_width_does_not_divide_the_readers_grid_fails_its_edge() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 30); + query(&mut memo, readout, None); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(15_000), entry(10_000)]); + let memo = &logical.memo; + assert_eq!( + chain_widths_serving(memo, state), + BTreeSet::from([5_000, 10_000, 15_000]) + ); + for edge in &logical.edges { + if let Some(width) = chain_width_below(memo, edge.producer_candidate_id.group) { + assert_eq!(width, 5_000, "{edge:?}"); + } + } + let mut live_widths = BTreeSet::new(); + for (index_in_group, merge) in candidates_of(memo, state).iter().enumerate() { + let Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) = &merge.expression.operator + else { + continue; + }; + let merge = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + match pruned_entry(&logical, merge) { + None => live_widths.extend(chain_width_below(memo, children[0])), + Some(pruned) => assert!( + pruned.causes.iter().any(|cause| matches!( + cause.root, + RootPruneReason::ChainWidthDoesNotDivideEvaluationGrid { + evaluation_period_ms: Some(5_000), + .. + } + )), + "{pruned:?}" + ), + } + } + assert_eq!(live_widths, BTreeSet::from([5_000])); +} + +/// A state under `@ T` adds no width; it reads a chain another query's +/// width created when that width divides T. +#[test] +fn an_absolute_anchor_reuses_only_existing_chains() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = range(&mut memo, input, 300); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + let mut pinned = Vec::new(); + // T = …040 s lies on the 60 s grid; T = …010 s does not. + for at_ms in [1_700_000_040_000, 1_700_000_010_000] { + let at = shift(&mut memo, input, 0, Some(AtModifier::Timestamp(at_ms))); + let at_window = range(&mut memo, at, 300); + let (state, root) = exact_state(&mut memo, at_window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + pinned.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let widths = |state| chain_widths_serving(&logical.memo, state); + assert_eq!(widths(state), BTreeSet::from([60_000])); + assert_eq!(widths(pinned[0]), BTreeSet::from([60_000])); + assert!(widths(pinned[1]).is_empty()); +} + +/// Chain groups come after the rollup +/// rule and the reach along candidate edges, so a state's widths come from +/// every query that reaches it through a candidate. +/// +/// A `sum by (job, instance) (count_over_time(m[5m]))` every 60 s; B +/// `sum by (job) (count_over_time(m[5m]))` every 90 s (both at phase 0). +/// B's state rolls up A's, so both reach A's state, whose widths are the +/// gcd closure of {60 s, 90 s} with 300 s: 30 s and 60 s. +#[test] +fn chain_groups_run_after_reach_so_a_rolled_up_query_adds_its_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job", "instance"], vec![]); + let window = range(&mut memo, input, 300); + let (by_job_instance, a) = exact_state( + &mut memo, + window, + ExactKind::Count, + Reduction::by(vec![2, 3]), + ); + let (by_job, b) = exact_state(&mut memo, window, ExactKind::Count, Reduction::by(vec![2])); + let query_a = query(&mut memo, a, Some(AccuracyTarget::exact())); + let query_b = query(&mut memo, b, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(90_000)]); + let memo = &logical.memo; + assert!( + candidates_of(memo, by_job).iter().any(|c| { + c.origin == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator.children() == vec![&by_job_instance] + }), + "B's state reads A's through a rollup" + ); + let reached_by = memo.groups()[by_job_instance.0 as usize].reached_by_queries(); + assert!( + reached_by.contains(&query_a) && reached_by.contains(&query_b), + "A's state is reached by A and B: {reached_by:?}" + ); + assert_eq!( + chain_widths_serving(memo, by_job_instance), + BTreeSet::from([30_000, 60_000]) + ); + assert_eq!(chain_widths_serving(memo, by_job), BTreeSet::from([30_000])); +} + +/// A window read at an `offset` takes the +/// offset into the gcd, since its windows' edges move by it. +/// `sum_over_time(m[5m] offset 45s)` every 60 s (phase 0): widths divide +/// 60 s, 300 s and 45 s, so 15 s only. +#[test] +fn an_offset_enters_the_gcd_of_the_widths() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let shifted = shift(&mut memo, input, 45, None); + let window = range(&mut memo, shifted, 300); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + chain_widths_serving(&logical.memo, state), + BTreeSet::from([15_000]) + ); +} + +/// A tumbling-window chain is a group +/// of its own, `SummaryTumblingWindowChain` over the rows with a width and +/// no range, offset or window count, so every range and offset of one rows +/// and width reads the same chain group through `TimeRange` (and +/// `TimeShift`) groups and a `SummaryMerge`. +/// +/// A `sum_over_time(m[5m])`, B `sum_over_time(m[5m] offset 1h)`, C +/// `sum_over_time(m[1h])`, each every 60 s (phase 0): one chain group Gw of +/// width 60 s over the scan of m; the groups `TimeShift(1h, Gw)`, +/// `TimeRange(5m, Gw)`, `TimeRange(5m, TimeShift(1h, Gw))` and +/// `TimeRange(1h, Gw)`; each state group has a `SummaryMerge` candidate +/// over its `TimeRange` group. +#[test] +fn one_range_free_chain_group_serves_every_range_and_offset() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let five_minutes = range(&mut memo, input, 300); + let shifted_input = shift(&mut memo, input, 3600, None); + let five_minutes_an_hour_ago = range(&mut memo, shifted_input, 300); + let one_hour = range(&mut memo, input, 3600); + let mut states = Vec::new(); + for window in [five_minutes, five_minutes_an_hour_ago, one_hour] { + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + states.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let memo = &logical.memo; + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let [chain] = chains[..] else { + panic!("one chain group, found {chains:?}"); + }; + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + tumbling_window_width_ms, + .. + }) = &memo.groups()[chain.0 as usize].expression + else { + unreachable!(); + }; + assert_eq!(*tumbling_window_width_ms, 60_000); + assert_eq!( + *child, input, + "the chain reads the scan: no range or shift below it" + ); + let range_over = |range_s: u64, child: MemoGroupId| { + group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(range_s), + kind: TimeRangeKind::Range, + child, + }), + ) + }; + let shifted_chain = group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: chain, + }), + ) + .expect("TimeShift(1h, Gw)"); + let ranges = [ + range_over(300, chain).expect("TimeRange(5m, Gw)"), + range_over(300, shifted_chain).expect("TimeRange(5m, TimeShift(1h, Gw))"), + range_over(3600, chain).expect("TimeRange(1h, Gw)"), + ]; + for (state, range) in states.iter().zip(ranges) { + assert!( + candidates_of(memo, *state).iter().any(|c| matches!( + &c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None }) + if *children == vec![range] + )), + "G{} reads G{} through a SummaryMerge", + state.0, + range.0 + ); + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs new file mode 100644 index 000000000..1dd877f87 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/dead_candidates.rs @@ -0,0 +1,118 @@ +//! Step 4: dead-candidate elimination and Pass 2's pruned list. +use std::collections::BTreeSet; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{CandidateForMemoGroupId, Memo, RootPruneReason}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::arithmetic; + +/// The concepts doc example "ratio of quantiles": +/// `quantile_over_time(0.99, m[5m]) / quantile_over_time(0.5, m[5m])`, +/// relative error 0.05, every 60 s; Pass 1 offers KLL and DDSketch to both +/// quantiles. The ratio carries only relative error, so each KLL readout is +/// pruned by the metric rule and every KLL build through it. The readouts +/// of DDSketch get the levels 0.02 … 0.001 (cap 0.045), so the builds with +/// alpha 0.1 and 0.05 miss every level they could serve. Chain groups are +/// added only for the surviving builds. +#[test] +fn dead_candidates_are_pruned_and_listed_before_chain_groups() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = range(&mut memo, input, 300); + let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &algorithms); + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &algorithms); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p99.result, p50.result), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let memo = &logical.memo; + let (kll_state, ddsketch_state) = (p99.by_algorithm[0].1, p99.by_algorithm[1].1); + // Each KLL readout: the metric rule at the ratio, directly. + for quantile in [&p99, &p50] { + let kll_readout = quantile.by_algorithm[0].2; + let pruned = pruned_entry(&logical, kll_readout).expect("the KLL readout is pruned"); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // Every KLL build: the metric rule on its edge into a KLL readout, + // whose output accepts only relative error. + for index_in_group in 0..parameter_list(SketchAlgorithm::Kll).len() { + let build = CandidateForMemoGroupId { + group: kll_state, + index_in_group, + }; + let pruned = pruned_entry(&logical, build).expect("every KLL build is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // DDSketch: alpha 0.1 and 0.05 pruned below every required accuracy; + // the levels of the readouts are 0.02 … 0.001. + let pruned_alphas: Vec = logical + .pruned + .iter() + .filter(|pruned| pruned.candidate.group == ddsketch_state) + .map(|pruned| { + assert!(pruned + .causes + .iter() + .all(|cause| cause.through_pruned_candidates.is_empty() + && matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. }))); + match memo + .candidate(pruned.candidate) + .logical_properties + .parameters() + { + Some(SketchParams::DDSketch { alpha }) => *alpha, + other => panic!("{other:?}"), + } + }) + .collect(); + assert_eq!(pruned_alphas, vec![0.1, 0.05]); + let readout_group = readout_of(memo, p99.by_algorithm[1].2).group; + let mut levels: Vec = requirements_of(memo, readout_group) + .into_iter() + .map(|required| required.expect("a DDSketch readout's level").max_error) + .collect(); + levels.sort_by(f64::total_cmp); + assert_eq!(levels, vec![0.001, 0.002, 0.005, 0.01, 0.02]); + // Chain builds exist only for the five surviving alphas. + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::DDSketch)) + }); + assert_eq!(chains.len(), 1); + assert_eq!(candidates_of(memo, chains[0]).len(), 5); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::Kll) + )) + .is_empty()); + // No logical edge names a pruned candidate. + let pruned: BTreeSet<_> = logical.pruned.iter().map(|p| p.candidate).collect(); + assert!(logical + .edges + .iter() + .all(|edge| !pruned.contains(&edge.producer_candidate_id) + && !pruned.contains(&edge.consumer_candidate_id))); + let _ = kll_state; +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs new file mode 100644 index 000000000..e12016d19 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/edges.rs @@ -0,0 +1,122 @@ +//! Step 6: the logical edges Pass 2 assembles and keeps ($\alpha^{\log}$ +//! and the metric rule themselves are tested in `asap_accuracy`). + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, RequiredAccuracy, RootPruneReason, +}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::arithmetic; + +/// Without accuracy targets (accuracy not enforced) readouts get no +/// requirement and no metric rule applies: every edge is admissible. +#[test] +fn without_targets_every_edge_is_admissible() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate( + &mut memo, + MemoGroupId(2), + SketchStatistic::Quantile { q: 0.5 }, + ); + let ratio = group(&mut memo, arithmetic(ArithmeticOpKind::Div, p99, p50)); + query(&mut memo, ratio, None); + let logical = pass2(memo, &[entry(60_000)]); + assert!(logical.pruned.is_empty()); + assert!(logical + .edges + .iter() + .all(|edge| edge.satisfies_logical_requirement)); +} + +/// Every candidate of an input group is an edge of each consumer input, +/// except that a candidate carrying a θ without building (a merge, a +/// `TimeRange` over a chain) reads only producers with its θ. +#[test] +fn every_candidate_of_an_input_group_is_an_edge_of_matching_parameters() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(300_000)]); + let listed = parameter_list(SketchAlgorithm::DDSketch).len(); + // The builds and one merge per θ over the 300 s chain. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); + let memo = &logical.memo; + for (index_in_group, merge) in candidates_of(memo, state).iter().enumerate() { + if !matches!( + merge.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) { + continue; + } + let consumer = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + let producers = edges_into(&logical, consumer, 0); + assert_eq!(producers.len(), 1); + assert_eq!( + memo.candidate(producers[0].0) + .logical_properties + .parameters(), + merge.logical_properties.parameters() + ); + } +} + +/// A merge carries the θ of the states it reads. +/// P = `quantile_over_time(0.9, m[5m])` at rank 0.005 keeps KLL builds +/// from k = 547; Q = `quantile_over_time(0.9, m[10m])` at rank 0.01 from +/// k = 269. Both read one 60 s chain, which has the builds of both, so P's +/// state gets `SummaryMerge[k=269]` and `SummaryMerge[k=547]`: the second +/// serves P's readout, the first is pruned with `BelowRequiredAccuracy`. +#[test] +fn a_merge_is_judged_by_the_parameters_it_carries() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, p_state, p_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let (_, _, q_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 600); + query(&mut memo, p_readout, Some(AccuracyTarget::max_error(0.005))); + query(&mut memo, q_readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let merge_with_k = |k: u32| -> CandidateForMemoGroupId { + let index_in_group = candidates_of(memo, p_state) + .iter() + .position(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) && c.logical_properties.parameters() == Some(&SketchParams::Kll { k }) + }) + .unwrap_or_else(|| panic!("SummaryMerge[k={k}] in P's state group")); + CandidateForMemoGroupId { + group: p_state, + index_in_group, + } + }; + let into_p = edges_into(&logical, own_candidate(p_readout), 0); + assert!(into_p.contains(&(merge_with_k(547), true)), "{into_p:?}"); + let pruned = pruned_entry(&logical, merge_with_k(269)).expect("SummaryMerge[k=269] is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + cause.root, + RootPruneReason::BelowRequiredAccuracy { + required: RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error, + .. + }, + .. + } if max_error == 0.005 + ))); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs new file mode 100644 index 000000000..aace59d8f --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/evaluation_grids.rs @@ -0,0 +1,114 @@ +//! The evaluation grid of each group, per query reaching it. +use std::collections::BTreeSet; +use std::time::Duration; + +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::summary_type::ExactKind; +use asap_types::workload::{ + DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, +}; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// A group's evaluation grid is the gcd of its reaching queries' grids; +/// `None` when one has no known grid. +#[test] +fn a_groups_evaluation_grid_is_the_gcd_of_its_queries_grids() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "m", &["job"], vec![]); + let window = range(&mut memo, input, 300); + let (_, a) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + let other = scan_group(&mut memo, "n", &["job"], vec![]); + query(&mut memo, a, None); + query(&mut memo, a, None); + query(&mut memo, other, None); + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(120_000), without_phase]); + let grid = |group: MemoGroupId| logical.evaluation_grid_of_group[group.0 as usize]; + assert_eq!(grid(window).map(|g| g.period_ms), Some(60_000)); + assert_eq!(grid(other), None); +} + +/// `max_over_time(sum_over_time(m[2m])[10m:])` read every 60 s +/// over a scan of `m`: the inner sum group and its state, the subquery +/// group and the query's root. +fn sum_over_time_in_a_subquery( + memo: &mut Memo, + resolution: Option, +) -> (MemoGroupId, MemoGroupId, MemoGroupId) { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = range(memo, input, 120); + let (state, sum) = exact_state(memo, window, ExactKind::Sum, Reduction::PerEntity); + let subquery = group( + memo, + Operator::NonASAP(NonASAPOp::PromqlSubquery { + range: Duration::from_secs(600), + resolution, + child: sum, + }), + ); + let root = group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: subquery, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Max { col: None }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ); + query(memo, root, None); + (state, subquery, root) +} + +/// Below a subquery `[10m:30s]` the inner expression is evaluated at the +/// multiples of 30 s, whatever the outer grid: read every 60 s, the inner +/// sum's state is on the 30 s grid and reads a 30 s chain, not a 60 s one +/// whose windows end only at every other inner evaluation. +#[test] +fn below_a_subquery_the_grid_is_its_resolution() { + let mut memo = Memo::new(&DataWorkload::default()); + let (state, subquery, root) = + sum_over_time_in_a_subquery(&mut memo, Some(Duration::from_secs(30))); + let logical = pass2(memo, &[entry(60_000)]); + let period = |group: MemoGroupId| { + logical.evaluation_grid_of_group[group.0 as usize].map(|grid| grid.period_ms) + }; + assert_eq!(period(root), Some(60_000)); + assert_eq!(period(subquery), Some(60_000)); + assert_eq!(period(state), Some(30_000)); + assert_eq!( + chain_widths_serving(&logical.memo, state), + BTreeSet::from([30_000]) + ); +} + +/// A subquery without a resolution steps at the server's default +/// evaluation interval, unknown to the planner: the groups below it have +/// no known grid and get no chain. +#[test] +fn below_a_subquery_without_a_resolution_there_is_no_grid() { + let mut memo = Memo::new(&DataWorkload::default()); + let (state, subquery, _) = sum_over_time_in_a_subquery(&mut memo, None); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + logical.evaluation_grid_of_group[subquery.0 as usize].map(|grid| grid.period_ms), + Some(60_000) + ); + assert_eq!(logical.evaluation_grid_of_group[state.0 as usize], None); + assert!(groups_where(&logical.memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + )) + .is_empty()); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs new file mode 100644 index 000000000..1327bb59b --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/exact_only_groups.rs @@ -0,0 +1,67 @@ +//! Step 3: the groups each query may read only exactly. +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::{ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; +use asap_types::stages::{ColumnLineage, Memo}; +use asap_types::summary_type::SketchAlgorithm; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::arithmetic; + +/// The metric rule is per query, not per group: Q0 = +/// `quantile_over_time(0.99, latency[5m]) / quantile_over_time(0.5, +/// latency[5m])` and Q1 = the same ratio `> 1`, both relative 0.05, share +/// the ratio group. Q0 may read the DDSketch readouts below it (the +/// division carries relative error); Q1's comparison has no contract, so +/// Q1 reads the ratio and every group below it only exactly. (Both read +/// the rows below a sketch build exactly: a build has no contract.) +#[test] +fn a_query_comparing_a_shared_ratio_reads_it_only_exactly() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "latency", &["job"], vec![]); + let window = range(&mut memo, input, 300); + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &[SketchAlgorithm::DDSketch]); + let ratio = group( + &mut memo, + arithmetic(ArithmeticOpKind::Div, p99.result, p50.result), + ); + let value = memo.groups()[ratio.0 as usize] + .group_properties + .column_lineage + .iter() + .position(|lineage| matches!(lineage, ColumnLineage::Computed { .. })) + .expect("the ratio computes its value"); + let above_one = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(value)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Promql, + }), + child: ratio, + }), + ); + let target = || AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + let q0 = query(&mut memo, ratio, Some(target())); + let q1 = query(&mut memo, above_one, Some(target())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let exact_only = &logical.exact_only_groups_of_query; + for group in [ratio, p99.result, p50.result, p99.by_algorithm[0].1] { + assert!(!exact_only[q0].contains(&group), "G{}", group.0); + } + for group in [ratio, p99.result, p50.result, p99.by_algorithm[0].1, window] { + assert!(exact_only[q1].contains(&group), "G{}", group.0); + } + assert!(!exact_only[q1].contains(&above_one)); + let ddsketch_p99 = readout_of(&logical.memo, p99.by_algorithm[0].2); + assert!( + pruned_entry(&logical, ddsketch_p99).is_none(), + "Q0 still reads the DDSketch readout" + ); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs b/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs new file mode 100644 index 000000000..625b0eb23 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/readout_requirements.rs @@ -0,0 +1,120 @@ +//! Step 3: readout requirements: what Pass 2 writes on the readouts and +//! prunes with the levels (the level math is tested in `asap_accuracy`). + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoQuery, RequiredAccuracy, RootPruneReason}; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// A single quantile query with rank error 0.01: its one readout gets that +/// requirement (λ = 1), and only the KLL builds whose rank error at δ = 1 +/// is within 0.01 satisfy it. +#[test] +fn a_single_readout_requires_the_query_target() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + requirements_of(&logical.memo, readout), + vec![Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error: 0.01, + failure_probability: 1.0, + })] + ); + let admitted: Vec = edges_into(&logical, own_candidate(readout), 0) + .into_iter() + .filter(|&(_, satisfies)| satisfies) + .filter_map(|(producer, _)| { + match logical + .memo + .candidate(producer) + .logical_properties + .parameters() + { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + } + }) + .collect(); + assert!(!admitted.is_empty()); + assert!(admitted.iter().all(|&k| k >= 269), "{admitted:?}"); + assert!(logical + .pruned + .iter() + .filter(|p| p.candidate.group == state) + .all(|p| p + .causes + .iter() + .all(|cause| matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. })))); +} + +/// A whole-result failure probability with the +/// number of values per evaluation unknown gives no per-value one, so the +/// readouts the query reaches are pruned with `ValuesPerEvaluationUnknown` +/// instead of requiring δ = 0. +#[test] +fn a_whole_result_failure_probability_without_values_per_evaluation_prunes_the_readout() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let query = memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: None, + }); + let logical = pass2(memo, &[entry(60_000)]); + let pruned = pruned_entry(&logical, own_candidate(readout)).expect("the readout is pruned"); + assert!(!pruned.causes.is_empty()); + for cause in &pruned.causes { + assert!( + matches!( + cause.root, + RootPruneReason::ValuesPerEvaluationUnknown { consumer, query: q } + if consumer == own_candidate(readout) && q == query + ), + "{cause:?}" + ); + } +} + +/// A readout another query gives a level is not pruned for a query whose +/// whole-result failure probability has no per-value one: Q0 (δ = 0.01, +/// values per evaluation unknown) and Q1 (no δ) read one KLL readout, and +/// the readout keeps Q1's level, live. Q0 reads its group, and the groups +/// below, only exactly: Q1's level says nothing about Q0's δ. Q1 may read +/// the readout and its state (only the rows below a build are exact for +/// both). +#[test] +fn a_readout_another_query_gives_a_level_survives_unknown_values_per_evaluation() { + let mut memo = Memo::new(&DataWorkload::default()); + let (window, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let q0 = memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + number_of_values_per_query_evaluation: None, + }); + let q1 = query(&mut memo, readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + assert_eq!( + requirements_of(&logical.memo, readout), + vec![Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error: 0.01, + failure_probability: 1.0, + })] + ); + let pruned = pruned_entry(&logical, own_candidate(readout)); + assert!(pruned.is_none(), "{pruned:?}"); + let exact_only = &logical.exact_only_groups_of_query; + for group in [readout, state, window] { + assert!(exact_only[q0].contains(&group), "G{}", group.0); + } + for group in [readout, state] { + assert!(!exact_only[q1].contains(&group), "G{}", group.0); + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs new file mode 100644 index 000000000..9e1ab7502 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/rollup.rs @@ -0,0 +1,527 @@ +//! Step 1: rollup from the finest common group. + +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use std::time::Duration; + +use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, TimeRangeKind}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; +use asap_types::summary_type::parameter_list; +use asap_types::summary_type::{ExactKind, SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; +use asap_types::test_support::memo_fixtures::compare; +use asap_types::test_support::memo_fixtures::{aggregate, state_grouped_by}; + +/// Concepts doc Example 1: F = `quantile by (job, host)(latency[5m])`, C = +/// `quantile by (host)(latency{job="api"}[5m])`. Equal definitions, the +/// narrowing `job ∈ {api}` on a key of F, {host} ⊆ {job, host}: C's state +/// gets `SummaryMerge(by host)(Filter(job = "api", F))`, and F is the +/// finest common group (nothing is synthesized). +#[test] +fn a_filtered_coarse_sketch_state_merges_the_finer_states_through_a_compensating_filter() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let api = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 2, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )], + ); + let fine_window = range(&mut memo, all, 300); + let coarse_window = range(&mut memo, api, 300); + let fine = sketch_state( + &mut memo, + fine_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + coarse_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let groups_before = memo.groups().len(); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 0, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + ExprSemantics::Promql, + )), + child: fine, + }), + ) + .expect("Filter(job = \"api\", F)"); + assert!(candidates_of(memo, coarse).iter().any(|c| c.origin + == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator + == Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![1]), + }))); + // F reads nothing from C, and no finest common group was added: the + // memo grew by the readouts, the filter, and the chains only. + assert!(candidates_of(memo, fine) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::Rollup)); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryAgg { .. }) + )) + .iter() + .all(|&g| g == fine || g == coarse || g.0 as usize >= groups_before)); +} + +/// Concepts doc Example 3: Q1 = `quantile by (host)(latency[5m])`, Q2 = +/// `quantile by (job)(latency{host="xxx"}[5m])`. Neither is finer, so the +/// finest common group G* = `SummaryAgg[KLL, by job, host]` over Q1's rows +/// is synthesized with one build per listed θ, Q1's state merges it by +/// host and Q2's merges `Filter(host = "xxx", G*)` by job. +#[test] +fn a_finest_common_group_no_query_asks_for_serves_both() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let xxx = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 3, + CompareOpKind::Eq, + ScalarValue::Utf8("xxx".into()), + ExprSemantics::Promql, + )], + ); + let all_window = range(&mut memo, all, 300); + let xxx_window = range(&mut memo, xxx, 300); + let by_host = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let by_job = sketch_state( + &mut memo, + xxx_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [by_host, by_job] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let finest = group_of( + memo, + &state_grouped_by( + FieldDataType::Sketch(SketchAlgorithm::Kll), + Reduction::by(vec![2, 3]), + all_window, + ), + ) + .expect("G*: KLL by job, host over Q1's rows"); + assert_eq!( + candidates_of(memo, finest) + .iter() + .filter(|c| c.expression.operator == memo.groups()[finest.0 as usize].expression) + .count(), + parameter_list(SketchAlgorithm::Kll).len() + ); + let merges = |state: MemoGroupId| -> Vec> { + candidates_of(memo, state) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect() + }; + let listed = parameter_list(SketchAlgorithm::Kll).len(); + // One merge per θ of G*'s builds. + assert_eq!( + merges(by_host), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![finest], + group_by: Some(vec![1]), + }); + listed + ] + ); + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("xxx".into()), + ExprSemantics::Promql, + )), + child: finest, + }), + ) + .expect("Filter(host = \"xxx\", G*)"); + assert_eq!(candidates_of(memo, filter).len(), listed); + assert_eq!( + merges(by_job), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![0]), + }); + listed + ] + ); +} + +/// A `quantile by (job)` over `status = "500"` rows does not roll up from +/// `quantile by (job, host)` over every row: `status` is no key of the +/// finer state, whose states mix both statuses. +#[test] +fn a_narrowing_on_a_column_that_is_no_key_gives_no_rollup() { + let mut memo = Memo::new(&DataWorkload::default()); + let labels = ["job", "host", "status"]; + let all = scan_group(&mut memo, "latency", &labels, vec![]); + let errors = scan_group( + &mut memo, + "latency", + &labels, + vec![compare( + 4, + CompareOpKind::Eq, + ScalarValue::Utf8("500".into()), + ExprSemantics::Promql, + )], + ); + let all_window = range(&mut memo, all, 300); + let errors_window = range(&mut memo, errors, 300); + let fine = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + errors_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + assert!(candidates_of(&logical.memo, coarse).iter().all(|c| !c + .expression + .operator + .children() + .contains(&&fine))); +} + +/// Concepts doc Example 2, exact values: `sum by (job)` rolls up the +/// finalized `sum by (job, instance)` of the same per-series rows with a +/// sum over its sum column. +#[test] +fn a_coarse_exact_sum_rolls_up_the_finer_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let window = range(&mut memo, input, 60); + let rate = group( + &mut memo, + state_grouped_by( + FieldDataType::exact(ExactKind::Rate), + Reduction::PerEntity, + window, + ), + ); + let rows = group( + &mut memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: rate }), + ); + let sum_by = |memo: &mut Memo, keys: Vec, name: &str| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: rows, + reduction: Reduction::by(keys), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![name.into()], + filters: vec![None], + having: None, + }), + ) + }; + let fine = sum_by(&mut memo, vec![2, 3], "value"); + let coarse = sum_by(&mut memo, vec![2], "value"); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let rollups: Vec<_> = candidates_of(&logical.memo, coarse) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect(); + assert_eq!( + rollups, + vec![Operator::NonASAP(NonASAPOp::Aggregate { + child: fine, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec!["value".into()], + filters: vec![None], + having: None, + })] + ); +} + +/// The instant vector `x` over `scan`: its latest sample per series. +fn instant(memo: &mut Memo, scan: MemoGroupId) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(1), + kind: TimeRangeKind::Instant, + child: scan, + }), + ) +} + +/// A count rolls up as a sum of counts, under the count's own column name: +/// `count by (job)` of PromQL (its measure unnamed, `count`) rolls up the +/// finer `count by (job, instance)` with `sum` over its count column, named +/// `count` as the coarse group's. +#[test] +fn a_coarse_count_rolls_up_the_finer_count_as_a_sum() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let rows = instant(&mut memo, input); + let fine = group( + &mut memo, + aggregate(Reduction::by(vec![2, 3]), AggIntent::Count, rows), + ); + let coarse = group( + &mut memo, + aggregate(Reduction::by(vec![2]), AggIntent::Count, rows), + ); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let rollups: Vec<_> = candidates_of(&logical.memo, coarse) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect(); + assert_eq!( + rollups, + vec![Operator::NonASAP(NonASAPOp::Aggregate { + child: fine, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec!["count".into()], + filters: vec![], + having: None, + })] + ); +} + +/// A PromQL `!=` matcher compensates with PromQL's comparison: +/// `sum by (job)` over `x{instance!="i1"}` rolls up `sum by (job, +/// instance)` over `x` through `Filter(instance != "i1")` with PromQL +/// semantics (a series without the label matches), not a negated SQL +/// `IN` list. +#[test] +fn a_not_equal_matcher_compensates_with_promql_semantics() { + let mut memo = Memo::new(&DataWorkload::default()); + let not_equal = |column: usize| { + compare( + column, + CompareOpKind::Ne, + ScalarValue::Utf8("i1".into()), + ExprSemantics::Promql, + ) + }; + let all = scan_group(&mut memo, "x", &["job", "instance"], vec![]); + let not_i1 = scan_group(&mut memo, "x", &["job", "instance"], vec![not_equal(3)]); + let all_rows = instant(&mut memo, all); + let not_i1_rows = instant(&mut memo, not_i1); + let fine = group( + &mut memo, + aggregate( + Reduction::by(vec![2, 3]), + AggIntent::Sum { col: None }, + all_rows, + ), + ); + let coarse = group( + &mut memo, + aggregate( + Reduction::by(vec![2]), + AggIntent::Sum { col: None }, + not_i1_rows, + ), + ); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let filters = groups_where( + &logical.memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == fine), + ); + let [filter] = filters[..] else { + panic!("one compensating filter: {filters:?}"); + }; + assert_eq!( + logical.memo.groups()[filter.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(not_equal(1)), + child: fine, + }) + ); +} + +/// A scan of the SQL table `flows(src, dst, bytes)` with `predicates`. +fn flows(memo: &mut Memo, predicates: Vec>) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates, + schema: Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("dst", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }), + ) +} + +/// A compensating filter compares with the semantics of the frontend of +/// the source it reads: `SUM(bytes) WHERE dst = 'x' GROUP BY src` over a +/// SQL table rolls up the exact sum state by (src, dst) through +/// `Filter(dst = 'x')` with SQL semantics, not PromQL's. +#[test] +fn a_compensating_filter_over_a_sql_table_has_sql_semantics() { + let mut memo = Memo::new(&DataWorkload::default()); + let all = flows(&mut memo, vec![]); + let x = flows( + &mut memo, + vec![Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("x".into()), + ExprSemantics::Sql, + ))], + ); + let sum_state = |memo: &mut Memo, rows: MemoGroupId, keys: Vec| { + group( + memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + summary_type: FieldDataType::exact(ExactKind::Sum), + input: SummaryUpdate::column(ColumnRef::Named("bytes".into())), + reduction: Reduction::by(keys), + filter: None, + }), + ) + }; + let fine = sum_state(&mut memo, all, vec![0, 1]); + let coarse = sum_state(&mut memo, x, vec![0]); + for state in [fine, coarse] { + let root = group( + &mut memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + ); + query(&mut memo, root, Some(AccuracyTarget::exact())); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let filters = groups_where( + &logical.memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == fine), + ); + let [filter] = filters[..] else { + panic!("one compensating filter: {filters:?}"); + }; + assert_eq!( + logical.memo.groups()[filter.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(compare( + 1, + CompareOpKind::Eq, + ScalarValue::Utf8("x".into()), + ExprSemantics::Sql + )), + child: fine, + }) + ); +} + +/// A rollup the memo rejects adds nothing: `COUNT(*) WHERE src = 'a'` (one +/// row, 0 when no row matches) does not roll up from `COUNT(*) GROUP BY +/// src` (a sum of counts, NULL over no row), and no compensating +/// `Filter(src = 'a')` group is left without a reader. +#[test] +fn a_rejected_rollup_leaves_no_compensating_filter() { + let mut memo = Memo::new(&DataWorkload::default()); + let all = flows(&mut memo, vec![]); + let a = flows( + &mut memo, + vec![Predicate(compare( + 0, + CompareOpKind::Eq, + ScalarValue::Utf8("a".into()), + ExprSemantics::Sql, + ))], + ); + let count_by = |memo: &mut Memo, rows: MemoGroupId, keys: Vec| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: rows, + reduction: Reduction::by(keys), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ) + }; + let by_src = count_by(&mut memo, all, vec![0]); + let only_a = count_by(&mut memo, a, vec![]); + query(&mut memo, by_src, Some(AccuracyTarget::exact())); + query(&mut memo, only_a, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + assert!(candidates_of(memo, only_a) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::Rollup)); + let roots = memo.root_groups(); + let orphans: Vec = (0..memo.groups().len()) + .map(|index| MemoGroupId(index as u32)) + .filter(|g| !roots.contains(g) && memo.groups()[g.0 as usize].parent_groups().is_empty()) + .collect(); + assert!(orphans.is_empty(), "groups without a reader: {orphans:?}"); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs b/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs new file mode 100644 index 000000000..2577f535e --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/topk.rs @@ -0,0 +1,46 @@ +//! Step 2: top-k reuse. + +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; +use asap_types::workload::DataWorkload; + +use crate::test_fixtures::*; +use asap_types::test_support::memo_builders::*; + +/// `topk(5, x)` reads `topk(10, x)`'s ranked rows: the 5 highest of the 10 +/// highest are the 5 highest. +#[test] +fn a_smaller_topk_reads_the_largest_topk_of_the_same_input() { + let mut memo = Memo::new(&DataWorkload::default()); + let input = scan_group(&mut memo, "x", &["job"], vec![]); + let topk = |memo: &mut Memo, k: usize| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: input, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::TopK { k }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ) + }; + let top5 = topk(&mut memo, 5); + let top10 = topk(&mut memo, 10); + query(&mut memo, top5, Some(AccuracyTarget::exact())); + query(&mut memo, top10, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let reuse: Vec<_> = candidates_of(&logical.memo, top5) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::TopKReuse) + .collect(); + assert_eq!(reuse.len(), 1); + assert_eq!(reuse[0].expression.operator.children(), vec![&top10]); + assert!(candidates_of(&logical.memo, top10) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::TopKReuse)); +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 6452f0bed..978ca7c0e 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -30,7 +30,7 @@ pub use report::{ /// `PhysicalASAPDAG` node per selected execution, its inputs from the /// selected edges, each node's `selected_physical_candidate` set and each /// materialized view node's `view_coverage` from its memo group -/// (`asap_types::stages::view_coverage`). +/// (`asap_types::plan::view_coverage`). /// `entries[q]` is query `q`'s workload entry; `data` the data workload /// with its bounds. /// @@ -46,5 +46,5 @@ pub fn select_plan( .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; let _ = (physical, entries, data); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::stages::view_coverage(memo, its group)") + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::plan::view_coverage(memo, its group)") } diff --git a/crates/types/src/accuracy/guarantee.rs b/crates/types/src/accuracy/guarantee.rs index e432b41df..156761594 100644 --- a/crates/types/src/accuracy/guarantee.rs +++ b/crates/types/src/accuracy/guarantee.rs @@ -1,24 +1,18 @@ -//! Machine-readable accuracy guarantees for finalized post-ASAP values -//! (issue #172). -//! -//! A selected post-ASAP plan used to carry no statement about the error of -//! the value it produces: every approximate layer was sized from its own -//! [`AccuracyTarget`] as if its input were exact, so an approximate parent -//! could silently consume an approximate child. This module is the -//! *vocabulary* that fixes that — the typed metric, the symbolic bound and -//! failure-probability expressions, the provenance trail, and the typed -//! rejection reasons. The *algebra* that composes these (the built-in -//! conservative rules and budget allocation) and the deployment's measured -//! accuracy (`AccuracyEvidence`) live one layer up in `asap_accuracy`: -//! this crate defines the shapes, the planning crates decide. +//! Machine-readable accuracy guarantees: the vocabulary for what error a +//! finalized, caller-visible value has — the typed metric +//! ([`AccuracyGuarantee`]), the symbolic bound and failure-probability +//! expressions ([`BoundExpr`], [`ProbabilityExpr`]) and the provenance trail +//! ([`GuaranteeSource`]). The theoretical guarantee of a summary +//! (`theoretical_guarantee`), each operator's error contract and the +//! deployment's measured accuracy (`AccuracyEvidence`) live one layer up in +//! `asap_accuracy`: this crate defines the shapes, the planning crates +//! decide. //! //! ## What a guarantee says //! -//! [`AccuracyGuarantee`] is attached to a finalized, caller-visible value — -//! [`crate::ir::OperatorNode::guarantee`] on a `SummaryEstimate` evaluation, an -//! exact accumulator, or a kept pre-ASAP sub-DAG — never to raw summary -//! state (a `SummaryAgg` sketch node carries `None`; its evaluation carries the -//! guarantee). Its statement is: +//! [`AccuracyGuarantee`] describes a finalized, caller-visible value — a +//! `SummaryEstimate` evaluation, an exact accumulator, or a kept pre-ASAP +//! sub-DAG — never raw summary state. Its statement is: //! //! ```text //! Pr[ err_metric(estimate, truth) > error ] <= failure_probability @@ -30,7 +24,7 @@ //! error. Metrics are **not** interchangeable: converting between them //! needs an explicit rule with evidence, never an implicit "compare the //! epsilons". [`AccuracyGuarantee`] shares its outer shape, [`AccuracySpec`], -//! with the requirement it is checked against, [`AccuracyTarget`]. +//! with the requirement it is checked against, [`AccuracyTarget`](super::AccuracyTarget). //! //! ## Why expressions, not numbers //! @@ -38,21 +32,20 @@ //! rather than bare `f64`s so a planning-time guarantee can reference a //! statistic it does not have (a group count, a stream's L1 norm) and stay //! honestly *unknown* until something instantiates it — a deployment's own -//! cardinality estimate, or a runtime posterior observation (issue #239). +//! cardinality estimate, or a runtime posterior observation. //! [`BoundExpr::evaluate`] returns `None`, never `0`, for such a bound; //! "unknown" and "zero" are different answers and a fail-closed planner //! treats them differently. //! //! ## What is deliberately *not* here //! -//! No `CorrectnessPolicy`-style enum: [`AccuracyTarget`] remains the one -//! authoritative requirement type, and [`AccuracyError`] is the typed reason -//! a candidate failed against it. No independence assumptions: the only +//! No `CorrectnessPolicy`-style enum: [`AccuracyTarget`](super::AccuracyTarget) remains the one +//! authoritative requirement type. No independence assumptions: the only //! probability combinator is the union bound. use serde::{Deserialize, Serialize}; -use super::target::{AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget}; +use super::target::{AccuracyMetric, AccuracyScope, AccuracySpec}; /// A symbolic, serializable error-bound expression. Non-negative real /// arithmetic only — there is no subtraction, so a bound can never be @@ -79,27 +72,6 @@ pub enum BoundExpr { } impl BoundExpr { - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - match self { - Self::Unknown { .. } => Self::Zero, - Self::Sum { terms } => Self::Sum { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - Self::Product { factors } => Self::Product { - factors: factors.iter().map(Self::optimistic_floor).collect(), - }, - Self::Scaled { factor, inner } => Self::Scaled { - factor: *factor, - inner: Box::new(inner.optimistic_floor()), - }, - Self::Max { terms } => Self::Max { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - other => other.clone(), - } - } - /// See [`AccuracyGuarantee::bind`]. pub(crate) fn bind(&self, value: &dyn Fn(&str) -> Option) -> Self { let all = |terms: &[BoundExpr]| terms.iter().map(|t| t.bind(value)).collect(); @@ -199,21 +171,6 @@ pub enum ProbabilityExpr { } impl ProbabilityExpr { - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - match self { - Self::Unknown { .. } => Self::Zero, - Self::UnionBound { terms } => Self::UnionBound { - terms: terms.iter().map(Self::optimistic_floor).collect(), - }, - Self::Scaled { count, inner } => Self::Scaled { - count: count.optimistic_floor(), - inner: Box::new(inner.optimistic_floor()), - }, - other => other.clone(), - } - } - /// Whether evaluation depends on a statistic not yet supplied. #[cfg(any(test, feature = "test-support"))] pub(crate) fn has_unknown(&self) -> bool { @@ -270,57 +227,9 @@ impl ProbabilityExpr { } } -/// How a parent operator consumes its inputs' values — the shape an -/// `propagate` rule is registered against. `#[non_exhaustive]` -/// for the same reason [`AccuracyMetric`] is. -#[non_exhaustive] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "op", rename_all = "snake_case")] -pub enum CompositionOperator { - /// Relative division with runtime finite/nonzero/range checks. For operand - /// bounds a,b the ratio bound is (a+b)/(1-b), with b < 1. - CheckedRelativeDivision, - /// An approximate summary built over its inputs' (approximate) values - /// — the sketch-over-sketch case. Its own `local` guarantee composes - /// with the inputs' under a same-metric rule. - ApproximateAggregate, - /// A deterministic transformation with an explicitly registered global - /// Lipschitz constant: `B_out ≤ constant · B_in + B_local`. The planner - /// never derives `constant` itself; only a caller that has proved it - /// may construct this operator. - Lipschitz { constant: f64 }, - /// An exact sum over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. - ExactSum, - /// An exact arithmetic mean over approximate values. For absolute-value - /// input guarantees, the output bound is the largest per-value bound. - ExactAverage, - /// An exact max/min over inputs carrying absolute-value guarantees — bounds the returned - /// *value* (`max` of the input bounds) but does not identify which key - /// is the true winner. - ExactExtremum, - /// Exact division of two relative-value estimates. For numerator bound - /// `a` and denominator bound `b < 1`, the output bound is - /// `(a + b) / (1 - b)`. - ExactDivision, - /// Exact multiplication of two estimates. For absolute errors `e_x`, - /// `e_y` the output bound is `|x|·e_y + |y|·e_x + e_x·e_y`, requiring - /// finite operand domains; for relative errors `a`, `b` it is - /// `(1 + a)(1 + b) − 1`. - ExactProduct, - /// PromQL `rate`: reset correction plus range-boundary extrapolation. - CounterRate, - /// PromQL `irate`: reset-aware slope over the final two samples. - InstantCounterRate, - /// PromQL `increase`: extrapolated reset-corrected increase over a range. - CounterIncrease, - /// A top-k selection over approximate inputs. Without a separated-margin - /// certificate the default model leaves membership accuracy unknown. - TopKSelection, -} - /// One entry in a [`AccuracyGuarantee`]'s provenance trail — enough for a /// reader to reconstruct *why* the bound is what it is without re-running -/// the planner. `#[non_exhaustive]` so runtime evidence (issue #239) and +/// the planner. `#[non_exhaustive]` so runtime evidence and /// deployment-specific sources can be appended later. #[non_exhaustive] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -332,8 +241,6 @@ pub enum GuaranteeSource { /// `"RetainedExact"`). reason: String, }, - /// The target this evaluation's sketch was sized against. - AccuracyTarget { target: AccuracyTarget }, /// The concrete sketch a evaluation's local guarantee was derived from. SketchEvaluation { algorithm: String, @@ -343,45 +250,8 @@ pub enum GuaranteeSource { params: serde_json::Value, query: String, }, - /// A composed input's own guarantee, carried verbatim so the trail is - /// self-contained. `input_index` is the input's position in the - /// composition (0-based, in the parent's child order). - ChildGuarantee { - input_index: usize, - guarantee: Box, - }, - /// The propagation rule that produced this guarantee from its inputs. - CompositionStep { - operator: CompositionOperator, - /// Stable rule name (e.g. `"additive_union_bound"`). - rule: String, - }, - /// The budget split that produced this layer's local target — present - /// only when an equal budget split re-sized a layer. - BudgetAllocation { - allocator: String, - layer: usize, - layer_count: usize, - local_target: AccuracyTarget, - end_to_end_target: AccuracyTarget, - }, - /// A statistic the bound needs but nothing supplied — the reason a - /// [`BoundExpr::Unknown`]/[`ProbabilityExpr::Unknown`] leaf exists. - UnavailableStatistic { statistic: String }, - /// Query-time evidence (issue #239's posterior bounds). Never produced - /// at planning time; reserved so a runtime can append its observation - /// to the same trail instead of inventing a parallel one. - RuntimeObservation { - source: String, - detail: serde_json::Value, - }, } -/// The statistic an unbound number of values per result is left as -/// ([`AccuracyGuarantee::over_scope`]); nothing binds it, so the -/// failure probability stays unknown. -pub const VALUES_PER_RESULT: &str = "values_per_result"; - /// The machine-readable accuracy statement attached to a finalized /// post-ASAP value — see the module docs for its semantics. Its /// [`AccuracySpec`] is symbolic, for plan selection to bind; a guarantee a @@ -427,17 +297,6 @@ impl AccuracyGuarantee { bound } - /// Optimistic lower bound (tests only): unknown non-negative - /// contributions count as zero, so it cannot certify a candidate; the - /// original guarantee remains unresolved. - #[cfg(test)] - fn optimistic_floor(&self) -> Self { - let mut floor = self.clone(); - floor.spec.error = self.spec.error.optimistic_floor(); - floor.spec.failure_probability = self.spec.failure_probability.optimistic_floor(); - floor - } - /// An unresolved bound or failure probability is not a certificate. #[cfg(any(test, feature = "test-support"))] pub fn has_unknown(&self) -> bool { @@ -459,116 +318,66 @@ impl AccuracyGuarantee { ) } - /// This guarantee stated over `scope`, for a result of - /// `values_per_result` values (`G`, `None` when unbound): a per-value - /// failure probability `δ` becomes `n · δ` by the union bound over the - /// `n` values the scope covers ([`AccuracyScope::max_values_covered`]), - /// whatever their dependence. The error is unchanged. A guarantee that - /// never fails (`δ = 0`) holds for every value at once; an unbound `n` - /// (the whole result of an unbound `G`) leaves the failure probability - /// symbolic in [`VALUES_PER_RESULT`]. A guarantee over any other scope - /// is returned as it is. Plan selection uses it to check a target over - /// `scope`. - pub fn over_scope(&self, scope: AccuracyScope, values_per_result: Option) -> Self { - let mut over = self.clone(); - // Only a per-value statement extends to more values by the union - // bound: one over some groups says nothing about the others, so it - // is returned as it is and misses a scope it does not cover. - if self.spec.accuracy_scope != AccuracyScope::PerValue || scope == AccuracyScope::PerValue { - return over; - } - over.spec.accuracy_scope = scope; - if !self.spec.failure_probability.is_zero() { - over.spec.failure_probability = ProbabilityExpr::Scaled { - count: scope.max_values_covered(values_per_result).map_or_else( - || BoundExpr::Unknown { - statistic: VALUES_PER_RESULT.into(), - }, - |values| BoundExpr::Constant { - value: values as f64, - }, - ), - inner: Box::new(self.spec.failure_probability.clone()), - }; - } - over - } - /// `true` iff this guarantee promises zero error with certainty. pub fn is_exact(&self) -> bool { self.spec.error.is_zero() && self.spec.failure_probability.is_zero() } - - /// How many approximate sketch evaluations contributed to this value — - /// `1` for a plain evaluation, `0` for an exact value, and the transitive - /// count through every [`GuaranteeSource::ChildGuarantee`] for a - /// composition. - #[cfg(any(test, feature = "test-support"))] - pub fn approximate_layer_count(&self) -> usize { - self.provenance - .iter() - .map(|source| match source { - GuaranteeSource::SketchEvaluation { .. } => 1, - GuaranteeSource::ChildGuarantee { guarantee, .. } => { - guarantee.approximate_layer_count() - } - _ => 0, - }) - .sum() - } -} - -/// Why an accuracy check rejected a candidate. Typed, serializable, and -/// carried through to DAG export so a rejection is as inspectable as a -/// selection. Never a reason to "treat the child as exact". -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, thiserror::Error)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum AccuracyError { - /// No registered propagation rule covers this operator over these - /// input metrics (or this local metric). - #[error( - "unsupported accuracy composition: {operator:?} over inputs {input_metrics:?} \ - with local {local_metric:?} — {reason}" - )] - UnsupportedComposition { - operator: CompositionOperator, - input_metrics: Vec, - local_metric: Option, - reason: String, - }, - /// An approximate input carries no guarantee at all, so nothing can be - /// composed over it. - #[error("input {input_index} of {operator:?} carries no accuracy guarantee")] - MissingInputGuarantee { - operator: CompositionOperator, - input_index: usize, - }, - /// The composed guarantee does not satisfy the applicable - /// [`AccuracyTarget`]. `bound`/`failure_probability` are the evaluated - /// values when known. - #[error( - "composed guarantee ({metric:?}, bound {bound:?}, failure probability \ - {failure_probability:?}) does not satisfy {target:?}" - )] - TargetNotSatisfied { - metric: AccuracyMetric, - bound: Option, - failure_probability: Option, - target: AccuracyTarget, - }, - /// No budget allocation could make the composition legal under the - /// end-to-end target. - #[error("no legal accuracy-budget allocation for {target:?} across {layer_count} layers")] - NoLegalAllocation { - target: AccuracyTarget, - layer_count: usize, - }, } #[cfg(test)] mod tests { use super::*; + impl BoundExpr { + fn optimistic_floor(&self) -> Self { + match self { + Self::Unknown { .. } => Self::Zero, + Self::Sum { terms } => Self::Sum { + terms: terms.iter().map(Self::optimistic_floor).collect(), + }, + Self::Product { factors } => Self::Product { + factors: factors.iter().map(Self::optimistic_floor).collect(), + }, + Self::Scaled { factor, inner } => Self::Scaled { + factor: *factor, + inner: Box::new(inner.optimistic_floor()), + }, + Self::Max { terms } => Self::Max { + terms: terms.iter().map(Self::optimistic_floor).collect(), + }, + other => other.clone(), + } + } + } + + impl ProbabilityExpr { + fn optimistic_floor(&self) -> Self { + match self { + Self::Unknown { .. } => Self::Zero, + Self::UnionBound { terms } => Self::UnionBound { + terms: terms.iter().map(Self::optimistic_floor).collect(), + }, + Self::Scaled { count, inner } => Self::Scaled { + count: count.optimistic_floor(), + inner: Box::new(inner.optimistic_floor()), + }, + other => other.clone(), + } + } + } + + impl AccuracyGuarantee { + /// Optimistic lower bound (tests only): unknown non-negative + /// contributions count as zero, so it cannot certify a candidate; the + /// original guarantee remains unresolved. + fn optimistic_floor(&self) -> Self { + let mut floor = self.clone(); + floor.spec.error = self.spec.error.optimistic_floor(); + floor.spec.failure_probability = self.spec.failure_probability.optimistic_floor(); + floor + } + } + #[test] fn unknown_statistic_never_evaluates_to_zero() { let b = BoundExpr::Product { @@ -647,13 +456,6 @@ mod tests { assert_eq!(p.evaluate(), Some(1.0)); } - #[test] - fn exact_guarantee_is_zero_layers() { - let g = AccuracyGuarantee::exact("test"); - assert!(g.is_exact()); - assert_eq!(g.approximate_layer_count(), 0); - } - #[test] fn guarantee_round_trips_through_json() { let g = AccuracyGuarantee::new( @@ -670,9 +472,8 @@ mod tests { ProbabilityExpr::UnionBound { terms: vec![ProbabilityExpr::Constant { value: 0.01 }], }, - vec![GuaranteeSource::CompositionStep { - operator: CompositionOperator::Lipschitz { constant: 2.0 }, - rule: "lipschitz".into(), + vec![GuaranteeSource::Exact { + reason: "test".into(), }], ); let json = serde_json::to_value(&g).unwrap(); @@ -683,47 +484,6 @@ mod tests { assert_eq!(back, g); } - /// A per-value guarantee over the groups holding at least 5% of the - /// records fails with at most 20 times its per-value probability, even - /// when the number of values the result returns is unknown: at most 20 - /// groups can hold 5% each. Fewer values bound it lower. The error is - /// unchanged, and a scope the guarantee already covers keeps it as it is. - #[test] - fn a_record_share_scope_union_bounds_over_one_over_share_groups() { - let per_value = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: 0.05 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let five_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.05); - let over = per_value.over_scope(five_percent, None); - assert_eq!(over.spec.accuracy_scope, five_percent); - assert_eq!(over.spec.error, per_value.spec.error); - assert!((over.spec.failure_probability.evaluate().unwrap() - 0.02).abs() < 1e-12); - let few = per_value.over_scope(five_percent, Some(4)); - assert!((few.spec.failure_probability.evaluate().unwrap() - 0.004).abs() < 1e-12); - assert_eq!( - per_value.over_scope(AccuracyScope::PerValue, None), - per_value - ); - let whole = per_value.over_scope(AccuracyScope::WholeResult, Some(10)); - assert_eq!(whole.over_scope(five_percent, None), whole); - // A statement over the 5% groups says nothing about groups between - // 1% and 5%, nor about the whole result: it is not rescaled. - let one_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.01); - assert_eq!(over.over_scope(one_percent, None), over); - assert_eq!(over.over_scope(AccuracyScope::WholeResult, Some(10)), over); - assert_eq!( - per_value - .over_scope(AccuracyScope::WholeResult, None) - .spec - .failure_probability - .evaluate(), - None - ); - } - /// A supplied statistic replaces its `Unknown` leaves, in the bound and /// in the failure probability; any other statistic stays unknown. #[test] diff --git a/crates/types/src/accuracy/mod.rs b/crates/types/src/accuracy/mod.rs index 0bcce29fb..6cd2381ae 100644 --- a/crates/types/src/accuracy/mod.rs +++ b/crates/types/src/accuracy/mod.rs @@ -1,4 +1,4 @@ -//! #511 §2.2: accuracy targets and the error metrics they are stated in +//! Accuracy targets and the error metrics they are stated in //! ([`AccuracyTarget`], [`AccuracyMetric`]), and machine-readable accuracy //! guarantees ([`AccuracyGuarantee`]). //! @@ -7,10 +7,7 @@ mod guarantee; mod target; -pub use guarantee::{ - AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, VALUES_PER_RESULT, -}; +pub use guarantee::{AccuracyGuarantee, BoundExpr, GuaranteeSource, ProbabilityExpr}; pub use target::{ default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, }; diff --git a/crates/types/src/accuracy/target.rs b/crates/types/src/accuracy/target.rs index 1b3562729..73e4084a0 100644 --- a/crates/types/src/accuracy/target.rs +++ b/crates/types/src/accuracy/target.rs @@ -1,4 +1,4 @@ -//! Accuracy requirements and the error metrics they are stated in (#690). +//! Accuracy requirements and the error metrics they are stated in. //! //! A requirement and a guarantee share one outer shape, [`AccuracySpec`]: //! an error metric, an error ε, a failure probability δ and a scope. A @@ -59,6 +59,14 @@ impl AccuracyMetric { pub const ABSOLUTE_STREAM_L2: Self = Self::Absolute { scale: Some(NormScale::StreamL2), }; + /// Every metric, for a contract or a target read in each of them. + pub const ALL: [Self; 5] = [ + Self::ABSOLUTE, + Self::ABSOLUTE_STREAM_L1, + Self::ABSOLUTE_STREAM_L2, + Self::Relative, + Self::Rank, + ]; } /// The wire name of an [`AccuracyMetric`]. @@ -279,6 +287,15 @@ impl AccuracyTarget { pub fn is_exact(&self) -> bool { self.max_error == 0.0 && self.failure_probability == Some(0.0) } + + /// The metrics this target accepts an error in: its metric, every + /// metric when it names none. + pub fn metrics(&self) -> Vec { + AccuracyMetric::ALL + .into_iter() + .filter(|&metric| self.accuracy_metric.is_none_or(|named| named == metric)) + .collect() + } } /// The metric a query's result is most naturally stated in, from the @@ -289,7 +306,7 @@ impl AccuracyTarget { /// root, whose error is as naturally a rank as a relative one. /// /// Used only to size and to display: a target whose metric is not set -/// accepts a guarantee in any metric (#690). +/// accepts a guarantee in any metric. pub fn default_metric_for_root(root: &QueryRoot) -> Option { match root { QueryRoot::Operator(node) => node_metric(node), diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index 35d1be914..9b945d2cb 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -13,7 +13,6 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use crate::accuracy::AccuracyGuarantee; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; use crate::ir::schema::Schema; @@ -27,7 +26,6 @@ pub struct FlatNode { pub operator: Operator, pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, - pub(crate) guarantee: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -73,7 +71,6 @@ fn visit( operator, result_kind: node.result_kind, schema: node.schema.clone(), - guarantee: node.guarantee.clone(), }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index edac475a3..1b45b6fda 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -84,7 +84,7 @@ pub enum ASAPOp> { /// It has no range and no window count: a `TimeRange` (and /// `TimeShift`) over it selects the windows a query reads, and how many /// windows it retains is $h_v$, a Stage 3 choice. - // Written as `PaneChain` before #1163; the aliases read those documents. + // Written as `PaneChain` before the rename; the alias reads those documents. #[serde(alias = "PaneChain")] SummaryTumblingWindowChain { child: C, diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index b9b53b2a4..5da385dc3 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -1,4 +1,4 @@ -//! #511 §1: the operators of the unified DAG and the parameter types their +//! The operators of the unified DAG and the parameter types their //! payloads are built from. //! //! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 4f4a0a685..19910108e 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,6 +1,6 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee). +//! every traversal needs (output category and schema). //! Physical properties (timing, lifetime, schedule) are not on the node: //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). @@ -14,7 +14,6 @@ use super::arguments::Source; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; -use crate::accuracy::AccuracyGuarantee; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -117,14 +116,12 @@ impl Operator { /// node. /// /// `schema` and `result_kind` are derived from `operator` and its children -/// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). +/// at construction and retained. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, - pub guarantee: Option, } impl OperatorNode { @@ -145,7 +142,6 @@ impl OperatorNode { operator, result_kind, schema, - guarantee: None, } } @@ -156,11 +152,6 @@ impl OperatorNode { Self::new(operator).map(Rc::new) } - pub fn with_guarantee(mut self, guarantee: Option) -> Self { - self.guarantee = guarantee; - self - } - /// The build whose state this summary-state node carries /// ([`summary_build`](super::asap::summary_build)). pub fn summary_build(self: &Rc) -> Option { @@ -208,7 +199,6 @@ impl OperatorNode { /// structural schema metadata. Only names that override the old derived /// schema are retained, for either operator category. A change in output /// arity with such overrides needs an explicit new naming assignment. - /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 46061a86d..af33f377b 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,4 +1,4 @@ -//! #511 §2.2–2.3: the planning properties a node carries besides its +//! The planning properties a node carries besides its //! operator and schema (when its value is produced), and the selection //! algebra aggregate coverage is stated in. Aggregate coverage itself is a //! memo-group property, derived and stored by the memo (`crate::stages`). diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 06b7cc02c..3db26cf21 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -167,7 +167,7 @@ mod limit_bound { /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. -pub(crate) fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { +pub fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { let literal = |e: &ScalarExpr| match e { ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), _ => None, @@ -443,8 +443,8 @@ impl Constraint { } } -#[cfg(any(test, feature = "test-support"))] impl SelectionBox { + #[cfg(any(test, feature = "test-support"))] pub fn new(columns: BTreeMap) -> Self { Self { columns } } @@ -453,9 +453,7 @@ impl SelectionBox { pub fn columns(&self) -> &BTreeMap { &self.columns } -} -impl SelectionBox { /// Whether no row lies in both boxes: some column is restricted by both /// and the restrictions are disjoint. pub(crate) fn disjoint(&self, other: &Self) -> bool { diff --git a/crates/types/src/ir/schema/series_identity.rs b/crates/types/src/ir/schema/series_identity.rs index 61bc53e69..3edf058b7 100644 --- a/crates/types/src/ir/schema/series_identity.rs +++ b/crates/types/src/ir/schema/series_identity.rs @@ -90,9 +90,7 @@ pub fn with_promql_series_identity( ) => {} _ => return Err("operator has no dynamic series-identity realization".into()), } - let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; - rebuilt.guarantee = node.guarantee.clone(); - let rebuilt = Rc::new(rebuilt); + let rebuilt = Rc::new(OperatorNode::new(operator).map_err(|e| e.to_string())?); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) } diff --git a/crates/types/src/plan/mod.rs b/crates/types/src/plan/mod.rs index c5e16eae8..8729cbcf9 100644 --- a/crates/types/src/plan/mod.rs +++ b/crates/types/src/plan/mod.rs @@ -3,3 +3,6 @@ //! ([`materialized_view`]). pub mod materialized_view; pub mod physical_asap_dag; +mod view_coverage; + +pub use view_coverage::view_coverage; diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 72fa6f2c6..23ffb001a 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -23,7 +23,6 @@ use crate::ir::operator::node::OperatorNode; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::scalar::ScalarExpr; -use crate::accuracy::AccuracyGuarantee; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::selection::SelectionBox; use crate::ir::schema::Schema; @@ -82,7 +81,6 @@ pub struct PhysicalASAPDAGNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub parameters: Option, pub output_schema: Schema, - pub guarantee: Option, /// For a materialized view (a node whose lifetime materializes its output /// across evaluations), how far back from the newest evaluation /// time its output must stay available once refreshed; for a query-time @@ -119,7 +117,7 @@ pub struct ViewCoverage { /// coverage's `AggregateDefinition` with every input group expanded /// recursively to its expression, down to the `Scan`s, holding no /// memo group or plan node ids, canonically serialized and hashed - /// (`stages::view_coverage`). + /// (`plan::view_coverage`). /// Equal across plans and processes that compute the same thing, /// whatever the selection, and different when they do not (a rollup to /// a coarser grouping computes something else and has its own). Inside @@ -303,9 +301,8 @@ impl PhysicalASAPDAG { .collect()) } - /// Every node of this DAG as IR, by node id, with its exported - /// guarantee (an operator node has no physical properties): what - /// [`Self::query_roots`] reads. Shared inputs stay shared. Validates the + /// Every node of this DAG as IR, by node id (an operator node has no + /// physical properties): what [`Self::query_roots`] reads. Shared inputs stay shared. Validates the /// DAG first. #[cfg(any(test, feature = "test-support"))] fn operator_nodes( @@ -331,8 +328,7 @@ impl PhysicalASAPDAG { let operator = node .payload .map_children(|child| Rc::clone(&rebuilt[child])); - let restored = OperatorNode::with_schema(operator, node.output_schema.clone()) - .with_guarantee(node.guarantee.clone()); + let restored = OperatorNode::with_schema(operator, node.output_schema.clone()); rebuilt.insert(id, Rc::new(restored)); } } @@ -652,7 +648,6 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( lifetime: Lifetime::None, parameters: None, output_schema: flat_node.schema, - guarantee: flat_node.guarantee, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/types/src/stages/logical/view_coverage.rs b/crates/types/src/plan/view_coverage.rs similarity index 94% rename from crates/types/src/stages/logical/view_coverage.rs rename to crates/types/src/plan/view_coverage.rs index 06b2bebc8..f640ff1fa 100644 --- a/crates/types/src/stages/logical/view_coverage.rs +++ b/crates/types/src/plan/view_coverage.rs @@ -1,6 +1,6 @@ //! A memo group's view coverage ([`view_coverage`]): the plan's content id //! for the group's aggregate coverage, which names no group. The memo -//! derives aggregate coverage ([`super::aggregate_coverage`]) over memo +//! derives aggregate coverage ([`AggregateCoverage`](crate::stages::AggregateCoverage)) over memo //! groups and stores it per group (`GroupProperties::aggregate_coverage`); //! this file expands the stored coverage's input groups and hashes the //! result. @@ -43,7 +43,7 @@ pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option { } /// An aggregate definition -/// ([`AggregateDefinition`](super::AggregateDefinition)) with its input +/// ([`AggregateDefinition`](crate::stages::AggregateDefinition)) with its input /// groups expanded ([`expanded`]): the fields in the same order, so it /// serializes as the definition would with each group replaced. #[derive(Serialize)] diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs deleted file mode 100644 index fd561c7e6..000000000 --- a/crates/types/src/stages/logical/candidate.rs +++ /dev/null @@ -1,282 +0,0 @@ -//! Candidate scope: each group's candidates $v = (e_v, p^{\log}_v)$ -//! (concepts doc `memo-groups-properties-requirements.md`, section 3; -//! #1203). - -use crate::ir::operator::asap::summary_type_of_operator; -use crate::ir::schema::FieldDataType; -use crate::ir::Operator; -use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; -use crate::summary_type::{parameter_list, SketchParams}; - -/// One candidate: position `index_in_group` in group `group`'s candidates -/// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] -pub struct CandidateForMemoGroupId { - pub group: MemoGroupId, - pub index_in_group: usize, -} - -/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. -#[derive(Debug, Clone)] -pub struct CandidateForMemoGroup { - /// $e_v$. - pub expression: CandidateExpression, - /// $p^{\log}_v$. - pub logical_properties: LogicalProperties, - /// $r^{\log}_{c,i}$, one per input group: per child of - /// `expression.operator`, in [`Operator::children`] order. Changed only - /// through [`Memo::set_candidate_input_requirements`](crate::stages::Memo::set_candidate_input_requirements). - pub input_requirements: Vec, - /// A fused candidate computes, as one operator, a sub-DAG pattern of - /// several operators in one appearance of the sub-expression (a top-k - /// over an aggregate as one heap summary; a sum over a sum as one - /// aggregate): the groups of the operators it replaces. Empty otherwise. - /// Reuse across appearances is Pass 2's, not fusion (concepts doc §3, - /// "Fused groups"). - pub fused_groups: Vec, - /// Which rule added it. Not in the formulation: kept for - /// `stage_pipeline --text` and the viewer (approved 2026-11-05). - pub origin: OriginOfCandidateForMemoGroup, -} - -/// Which rule added a candidate. Nothing in planning reads it. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] -pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary type. - Pass1, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, - /// A tumbling-window chain, or a merge reading one, of a summary state group. - TumblingWindows, - /// A coarse group regrouping a finer group. - Rollup, - /// A top-k read from another group's ranking. - TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, -} - -/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with -/// its arguments, whose children are groups. The input groups $G_{v,i}$ are -/// [`Operator::children`], in that order. -#[derive(Debug, Clone, PartialEq)] -pub struct CandidateExpression { - pub operator: Operator, -} - -/// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. -#[derive(Debug, thiserror::Error)] -pub enum CandidateError { - /// The candidate's expression is not valid over its input groups (its - /// output schema does not derive, an input fails validation, or a merge - /// reads states it may not merge): a bug in the rule that built it. - #[error("the candidate's expression is not valid over its input groups: {0}")] - NotValidOverInputGroups(MemoGroupError), - /// Its output schema or output kind differs from the group's: it does - /// not compute the group's result (a bug in the rule that built it). - #[error("the candidate's output differs from group {group:?}'s")] - OutputDiffersFromGroup { group: MemoGroupId }, -} - -/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's -/// argument, not a property, and a tumbling-window chain's width is the -/// chain operator's argument. [`Self::new`] rejects parameters of another -/// algorithm, not in the algorithm's list, missing on a sketch build (a chain -/// included) or present on an exact operator. -#[derive(Debug, Clone)] -pub struct LogicalProperties { - /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for - /// an exact operator. - pub(super) parameters: Option, -} - -impl LogicalProperties { - pub fn new( - expression: &CandidateExpression, - parameters: Option, - ) -> Result { - let algorithm = match summary_type_of_operator(&expression.operator) { - Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), - _ => None, - }; - match (algorithm, ¶meters) { - (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), - (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), - (Some(algorithm), Some(parameters)) => { - if !parameter_list(algorithm).contains(parameters) { - return Err(LogicalPropertiesError::ParametersNotInList); - } - } - (None, None) => {} - } - Ok(Self { parameters }) - } - - /// $\boldsymbol\theta_v$. - pub fn parameters(&self) -> Option<&SketchParams> { - self.parameters.as_ref() - } -} - -/// Logical properties that do not fit their operator: bad input to -/// [`LogicalProperties::new`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum LogicalPropertiesError { - #[error("an exact operator has no summary parameters")] - ParametersOnExactOperator, - #[error("a sketch build needs parameters from its algorithm's list")] - SketchBuildWithoutParameters, - #[error("the parameters are not in their algorithm's predefined list")] - ParametersNotInList, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::ASAPOp; - use crate::summary_type::SketchAlgorithm; - - use crate::ir::operator::arguments::Reduction; - - use crate::ir::scalar::expr_ir::ColumnRef; - - use crate::ir::schema::FieldDataType; - use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; - - /// `SummaryAgg` over group 0 building a state of `summary_type`. - fn summary_agg(summary_type: FieldDataType) -> CandidateExpression { - CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryAgg { - child: MemoGroupId(0), - summary_type, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }), - } - } - - const KLL_269: SketchParams = SketchParams::Kll { k: 269 }; - - const CMS: SketchParams = SketchParams::Cms { - width: 272, - depth: 5, - }; - - const HYDRA_CMS: SketchParams = SketchParams::Hydra(HydraParams::HydraCms { - width: 272, - depth: 5, - shared_rows: 3, - shared_columns: 1_024, - }); - - /// `LogicalProperties::new` over every operator kind and parameters: a - /// sketch build (a chain included) needs parameters of its algorithm from - /// its list; parameters are rejected on an operator that builds no - /// sketch. - #[test] - fn logical_properties_new_over_every_operator_and_parameters() { - use LogicalPropertiesError as Invalid; - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let kll_chain = CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: MemoGroupId(0), - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms: 5_000, - }), - }; - let readout = CandidateExpression { - operator: Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: MemoGroupId(0), - query: crate::summary_type::SketchStatistic::Quantile { q: 0.5 }, - }), - }; - // k = 270 is KLL's structure but not in its list. - let parameters = [ - None, - Some(KLL_269), - Some(CMS), - Some(HYDRA_CMS), - Some(SketchParams::Kll { k: 270 }), - ]; - let no_parameters = Err(Invalid::ParametersOnExactOperator); - let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); - let not_in_list = Err(Invalid::ParametersNotInList); - let exact = [ - Ok(()), - no_parameters, - no_parameters, - no_parameters, - no_parameters, - ]; - // Another algorithm's parameters are not in KLL's list either. - let kll_build = [ - needs_parameters, - Ok(()), - not_in_list, - not_in_list, - not_in_list, - ]; - // Per operator: the result for each of `parameters`. - let table = [ - (summary_agg(kll), kll_build), - // No Hydra θ is listed yet, so no Hydra build is valid. - ( - summary_agg(FieldDataType::Sketch(SketchAlgorithm::HydraCms)), - [ - needs_parameters, - not_in_list, - not_in_list, - not_in_list, - not_in_list, - ], - ), - (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), - (kll_chain, kll_build), - (readout, exact), - ]; - for (expression, by_parameters) in table { - for (parameters, expected) in parameters.iter().zip(by_parameters) { - let properties = LogicalProperties::new(&expression, parameters.clone()); - assert_eq!( - properties.as_ref().map(|_| ()).map_err(|e| *e), - expected, - "{:?} {parameters:?}", - expression.operator.kind_name() - ); - if let Ok(properties) = properties { - assert_eq!(properties.parameters(), parameters.as_ref()); - } - } - } - } - - /// `summary_type_of_operator` reads the summary type of a `SummaryAgg` - /// and of a chain, and of no other operator. - #[test] - fn summary_type_of_operator_reads_the_builds_summary_type() { - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: MemoGroupId(0), - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms: 5_000, - }); - let merge = Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![MemoGroupId(0)], - group_by: None, - }); - assert_eq!( - summary_type_of_operator(&summary_agg(kll.clone()).operator), - Some(&kll) - ); - assert_eq!(summary_type_of_operator(&chain), Some(&kll)); - assert_eq!(summary_type_of_operator(&merge), None); - } -} diff --git a/crates/types/src/stages/logical/candidate/mod.rs b/crates/types/src/stages/logical/candidate/mod.rs new file mode 100644 index 000000000..f003508db --- /dev/null +++ b/crates/types/src/stages/logical/candidate/mod.rs @@ -0,0 +1,171 @@ +//! Candidate scope: each group's candidates $v = (e_v, p^{\log}_v)$ +//! (concepts doc `memo-groups-properties-requirements.md`, section 3). + +use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::schema::FieldDataType; +use crate::ir::Operator; +use crate::stages::{LogicalRequirements, MemoGroupError, MemoGroupId}; +use crate::summary_type::{parameter_list, SketchAlgorithm, SketchParams}; + +/// One candidate: position `index_in_group` in group `group`'s candidates +/// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct CandidateForMemoGroupId { + pub group: MemoGroupId, + pub index_in_group: usize, +} + +/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. +#[derive(Debug, Clone)] +pub struct CandidateForMemoGroup { + /// $e_v$. + pub expression: CandidateExpression, + /// $p^{\log}_v$. + pub logical_properties: LogicalProperties, + /// $r^{\log}_{c,i}$, one per input group: per child of + /// `expression.operator`, in [`Operator::children`] order. Changed only + /// through [`Memo::set_candidate_input_requirements`](crate::stages::Memo::set_candidate_input_requirements). + pub input_requirements: Vec, + /// A fused candidate computes, as one operator, a sub-DAG pattern of + /// several operators in one appearance of the sub-expression (a top-k + /// over an aggregate as one heap summary; a sum over a sum as one + /// aggregate): the groups of the operators it replaces. Empty otherwise. + /// Reuse across appearances is Pass 2's, not fusion (concepts doc §3, + /// "Fused groups"). + pub fused_groups: Vec, + /// Which rule added it. Not in the formulation: kept for + /// `stage_pipeline --text` and the viewer (approved 2026-11-05). + pub origin: OriginOfCandidateForMemoGroup, +} + +impl CandidateForMemoGroup { + /// The candidate computing `operator` with θ `parameters`, added by + /// rule `origin`, fusing `fused_groups`. Its inputs have no requirement + /// yet: Pass 2 sets each readout's + /// (`Memo::set_candidate_input_requirements`). + pub fn new( + operator: Operator, + parameters: Option, + origin: OriginOfCandidateForMemoGroup, + fused_groups: Vec, + ) -> Self { + let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; + Self { + expression: CandidateExpression { operator }, + logical_properties: LogicalProperties::new(parameters), + input_requirements, + fused_groups, + origin, + } + } +} + +/// The θ of each build candidate of `build`: every listed θ +/// (`parameter_list`) of a sketch build, `[None]` for any other operator +/// (an exact build). A group gets one build candidate per entry. +pub fn listed_build_parameters(build: &Operator) -> Vec> { + match summary_type_of_operator(build) { + Some(FieldDataType::Sketch(algorithm)) => parameter_list(*algorithm) + .iter() + .cloned() + .map(Some) + .collect(), + _ => vec![None], + } +} + +/// Which rule added a candidate. Nothing in planning reads it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub enum OriginOfCandidateForMemoGroup { + /// Pass 1: exact, exact accumulator, or a summary type. + Pass1, + /// A tumbling-window chain, or a merge reading one, of a summary state group. + TumblingWindows, + /// A coarse group rolled up from a finer one, the compensating `Filter` + /// it reads, or a build of a finest common group. + Rollup, + /// A top-k read from another group's ranking. + TopKReuse, +} + +/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with +/// its arguments, whose children are groups. The input groups $G_{v,i}$ are +/// [`Operator::children`], in that order. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateExpression { + pub operator: Operator, +} + +/// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. +#[derive(Debug, thiserror::Error)] +pub enum CandidateError { + /// The candidate's expression is not valid over its input groups (its + /// output schema does not derive, an input fails validation, or a merge + /// reads states it may not merge): a bug in the rule that built it. + #[error("the candidate's expression is not valid over its input groups: {0}")] + NotValidOverInputGroups(MemoGroupError), + /// Its output schema or output kind differs from the group's: it does + /// not compute the group's result (a bug in the rule that built it). + #[error("the candidate's output differs from group {group:?}'s")] + OutputDiffersFromGroup { group: MemoGroupId }, + /// Its θ does not fit the group's output (a bug in the rule that built + /// it). + #[error("the candidate's parameters do not fit its group's output: {0}")] + LogicalProperties(LogicalPropertiesError), +} + +/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's +/// argument, not a property, and a tumbling-window chain's width is the +/// chain operator's argument. A candidate passing states on without building +/// them (`SummaryMerge`, and a `Filter`, `TimeRange` or `TimeShift` over +/// states) carries the θ of the states it reads. +/// [`Memo::add_candidate`](crate::stages::Memo::add_candidate) checks θ +/// against the candidate's group's output ([`parameters_fit`]). +#[derive(Debug, Clone)] +pub struct LogicalProperties { + /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for + /// an exact operator. + pub(super) parameters: Option, +} + +impl LogicalProperties { + pub fn new(parameters: Option) -> Self { + Self { parameters } + } + + /// $\boldsymbol\theta_v$. + pub fn parameters(&self) -> Option<&SketchParams> { + self.parameters.as_ref() + } +} + +/// Whether θ `parameters` fit an output of sketch `algorithm` (`None`: no +/// sketch state): from its list, and present exactly for a sketch. +pub(super) fn parameters_fit( + algorithm: Option, + parameters: Option<&SketchParams>, +) -> Result<(), LogicalPropertiesError> { + match (algorithm, parameters) { + (Some(_), None) => Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) if !parameter_list(algorithm).contains(parameters) => { + Err(LogicalPropertiesError::ParametersNotInList) + } + _ => Ok(()), + } +} + +/// θ that does not fit the output of a candidate's group, as +/// [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum LogicalPropertiesError { + #[error("an exact operator has no summary parameters")] + ParametersOnExactOperator, + #[error("a sketch build, or a candidate passing sketch states on, needs parameters from its algorithm's list")] + SketchBuildWithoutParameters, + #[error("the parameters are not in their algorithm's predefined list")] + ParametersNotInList, +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/logical/candidate/tests.rs b/crates/types/src/stages/logical/candidate/tests.rs new file mode 100644 index 000000000..749e628fa --- /dev/null +++ b/crates/types/src/stages/logical/candidate/tests.rs @@ -0,0 +1,60 @@ +use super::*; +use crate::ir::operator::arguments::Reduction; +use crate::ir::scalar::expr_ir::ColumnRef; +use crate::ir::ASAPOp; +use crate::summary_type::{ExactKind, SummaryUpdate}; + +/// `SummaryAgg` over group 0 building a state of `summary_type`. +fn summary_agg(summary_type: FieldDataType) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: MemoGroupId(0), + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +/// `summary_type_of_operator` reads the summary type of a `SummaryAgg` +/// and of a chain, and of no other operator. +#[test] +fn summary_type_of_operator_reads_the_builds_summary_type() { + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }); + assert_eq!( + summary_type_of_operator(&summary_agg(kll.clone())), + Some(&kll) + ); + assert_eq!(summary_type_of_operator(&chain), Some(&kll)); + assert_eq!(summary_type_of_operator(&merge), None); +} + +/// A sketch build has one entry per listed θ, an exact build one +/// without θ. +#[test] +fn listed_build_parameters_lists_a_sketch_builds_parameters() { + let kll = SketchAlgorithm::Kll; + assert_eq!( + listed_build_parameters(&summary_agg(FieldDataType::Sketch(kll))), + parameter_list(kll) + .iter() + .cloned() + .map(Some) + .collect::>() + ); + assert_eq!( + listed_build_parameters(&summary_agg(FieldDataType::exact(ExactKind::Sum))), + vec![None] + ); +} diff --git a/crates/types/src/stages/logical/edge.rs b/crates/types/src/stages/logical/edge.rs index 4c9896618..9ccb9f809 100644 --- a/crates/types/src/stages/logical/edge.rs +++ b/crates/types/src/stages/logical/edge.rs @@ -1,4 +1,4 @@ -//! Edge scope: the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). +//! Edge scope: the logical edges $(u, c, i)$ with $\alpha^{\log}$. use crate::stages::CandidateForMemoGroupId; diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/group/aggregate_coverage.rs similarity index 99% rename from crates/types/src/stages/logical/aggregate_coverage.rs rename to crates/types/src/stages/logical/group/aggregate_coverage.rs index a14e9552b..baef20aec 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/group/aggregate_coverage.rs @@ -26,7 +26,7 @@ use std::ops::Bound; use thiserror::Error; use super::column_properties::{project_lineage, ColumnLineage}; -use super::group::MemoGroupAsInput; +use super::MemoGroupAsInput; use crate::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; @@ -71,7 +71,6 @@ pub struct AggregateCoverage { pub(crate) selection: Vec, } -#[cfg(any(test, feature = "test-support"))] impl AggregateCoverage { pub fn definition(&self) -> &AggregateDefinition { &self.definition diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/group/column_properties.rs similarity index 99% rename from crates/types/src/stages/logical/column_properties.rs rename to crates/types/src/stages/logical/group/column_properties.rs index fe500ae94..82be7197e 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/group/column_properties.rs @@ -10,7 +10,7 @@ use std::collections::BTreeMap; -use super::group::{MemoGroupAsInput, MemoGroupId}; +use super::{MemoGroupAsInput, MemoGroupId}; use crate::ir::operator::arguments::{ BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, SourceColumn, }; diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group/mod.rs similarity index 91% rename from crates/types/src/stages/logical/group.rs rename to crates/types/src/stages/logical/group/mod.rs index 44fa2f849..12b5a9bf9 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -3,18 +3,26 @@ use std::collections::BTreeSet; -use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage, CoverageError}; -use super::column_properties::{column_lineage_and_signs, ColumnLineage, ValueSign}; +mod aggregate_coverage; +mod column_properties; + +pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; +pub use column_properties::{ColumnLineage, ValueSign}; + +use aggregate_coverage::aggregate_coverage; +use column_properties::column_lineage_and_signs; + use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; use crate::ir::scalar::column_resolution::ResolveError; use crate::ir::schema::topk::ranked_column; -use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::schema::{ColumnId, FieldDataType, Schema}; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; use crate::ir::{NonASAPOp, ScalarExpr}; use crate::ir::{Operator, OperatorResultKind}; use crate::stages::{CandidateForMemoGroup, Memo}; +use crate::summary_type::SketchAlgorithm; /// A group, by index in the memo's groups. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -92,6 +100,23 @@ pub struct GroupProperties { pub scanned_sources: BTreeSet, } +impl GroupProperties { + /// The sketch algorithm of the state the group outputs; `None` for + /// rows, values and exact states. + pub fn sketch_algorithm_of_state(&self) -> Option { + if self.result_kind != OperatorResultKind::State { + return None; + } + self.schema + .fields + .iter() + .find_map(|field| match field.dtype { + FieldDataType::Sketch(algorithm) => Some(algorithm), + _ => None, + }) + } +} + /// Why an expression is not valid over its input groups: what /// [`Memo::find_or_add_group`] and, wrapped in /// [`CandidateError::NotValidOverInputGroups`](crate::stages::CandidateError::NotValidOverInputGroups), diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing/mod.rs similarity index 89% rename from crates/types/src/stages/logical/memo/hash_consing.rs rename to crates/types/src/stages/logical/memo/hash_consing/mod.rs index 61c489187..266d88124 100644 --- a/crates/types/src/stages/logical/memo/hash_consing.rs +++ b/crates/types/src/stages/logical/memo/hash_consing/mod.rs @@ -60,15 +60,18 @@ pub(super) fn same_value(left: &T, right: &T) - } /// Whether `left` and `right` are one candidate of a group: the same -/// expression and logical properties (parameters), each by [`same_value`]: -/// typed equality first, serialized only when typed-equal. -pub(super) fn same_expression_and_logical_properties( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> bool { +/// expression and logical properties (parameters), each by [`same_value`] +/// (typed equality first, serialized only when typed-equal), and the same +/// input requirements, so two readouts of one state with different required +/// accuracy are two candidates. +pub(super) fn same_candidate(left: &CandidateForMemoGroup, right: &CandidateForMemoGroup) -> bool { same_value(&left.expression.operator, &right.expression.operator) && same_value( &left.logical_properties.parameters, &right.logical_properties.parameters, ) + && left.input_requirements == right.input_requirements } + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/logical/memo/hash_consing/tests.rs b/crates/types/src/stages/logical/memo/hash_consing/tests.rs new file mode 100644 index 000000000..07bfe87fd --- /dev/null +++ b/crates/types/src/stages/logical/memo/hash_consing/tests.rs @@ -0,0 +1,385 @@ +//! Hash-consing: equal expressions are one memo group, and a duplicate +//! candidate is the candidate it duplicates, on the concepts doc's KLL example +//! (section 1: G1 the latency scan, G2 its KLL state). + +use super::super::tests::kll_k; +use super::super::*; +use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{ProjectItem, SortKey}; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; +use crate::summary_type::SketchAlgorithm; +use crate::test_support::memo_builders::try_candidate; +use crate::test_support::memo_fixtures::{aggregate_with, scan, state}; + +/// An unordered `LIMIT 10` over `input`, whose copies may return +/// different rows. +fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) +} + +/// An equal expression over the same input groups is found, not added. +#[test] +fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + assert_eq!( + memo.find_or_add_group(scan("latency", &[], vec![])) + .unwrap(), + g1 + ); + assert_eq!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(), + g2 + ); + assert_ne!( + memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), + g1 + ); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); +} + +/// The same operator over different input groups is a different group. +#[test] +fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::new(&DataWorkload::default()); + let latency = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); + assert_ne!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) + .unwrap(), + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) + .unwrap() + ); +} + +/// An expression whose copies may return different rows (an unordered +/// `LIMIT`) gets a new group every time. +#[test] +fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); +} + +/// KLL builds with k = 269 and k = 547 are two candidates of one group +/// G2: θ is a logical property, not part of the expression. +#[test] +fn kll_builds_with_k_269_and_k_547_share_one_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let k269_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k269 = try_candidate( + &mut memo, + k269_group, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + let k547_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k547 = try_candidate( + &mut memo, + k547_group, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(547), + ) + .unwrap(); + assert_eq!(k269_group, k547_group); + assert_ne!(k269, k547); + assert_eq!(memo.groups()[k269_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k547).logical_properties.parameters(), + kll_k(547).as_ref() + ); +} + +/// A candidate with the same expression and logical properties as one +/// in the group is not added again; its id is returned. +#[test] +fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let first = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + let again = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(269), + ) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let other_size = try_candidate( + &mut memo, + g2, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(547), + ); + assert_ne!(other_size.unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); +} + +/// `Scan m`: `[ts, service, value, latency]`, no unique key. +fn service_scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +/// Whether `a` and `b`, each built into one memo through +/// `find_or_add_group`, are one group. +fn one_group( + a: impl FnOnce(&mut Memo) -> MemoGroupId, + b: impl FnOnce(&mut Memo) -> MemoGroupId, +) -> bool { + let mut memo = Memo::new(&DataWorkload::default()); + let a = a(&mut memo); + let b = b(&mut memo); + a == b +} + +/// `operator` over the group of [`service_scan`]. +fn over_scan( + operator: impl FnOnce(MemoGroupId) -> Operator, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + |memo| { + let scan = memo.find_or_add_group(service_scan()).unwrap(); + memo.find_or_add_group(operator(scan)).unwrap() + } +} + +/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. +fn quantile_agg( + by: Vec, + col: Option, + q: f64, + filter: Option>, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + aggregate_with( + Reduction::by(by), + vec![(AggIntent::Quantile { col, q }, filter)], + None, + child, + ) + }) +} + +/// Aggregates that differ in a column, a quantile or a filter are +/// different groups. +#[test] +fn aggregates_that_differ_in_an_argument_are_different_groups() { + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(3), 0.5, None) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let latency_above_1 = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Sql, + }); + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) + )); +} + +/// Equal expressions without a unique key are one group when they return +/// the same rows each time: a keyless scan, a global aggregate, a +/// `without` aggregate. +#[test] +fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { + let table_scan = |memo: &mut Memo| { + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + })) + .unwrap() + }; + assert!(one_group(table_scan, table_scan)); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let without = || { + over_scan(|child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }) + }; + assert!(one_group(without(), without())); +} + +/// `DISTINCT` on every column returns the same rows each time and is one +/// group; a `Dedup` on some columns keeps an unspecified row per key. +#[test] +fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); +} + +/// Two unordered `LIMIT 10`s may return different rows: two groups, and +/// so are equal parents over them. +#[test] +fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group( + over_scan(unordered_limit), + over_scan(unordered_limit) + )); + let distinct_over_limit = |memo: &mut Memo| { + let limit = over_scan(unordered_limit)(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { + cols: vec![], + child: limit, + })) + .unwrap() + }; + assert!(!one_group(distinct_over_limit, distinct_over_limit)); +} + +/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows +/// of a fixed order (one group); over a `Sort` partitioned differently, +/// which rows come first depends on input order (two groups). +#[test] +fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + move |memo: &mut Memo| { + let sorted = over_scan(|child| { + Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child, + }) + })(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: sorted, + })) + .unwrap() + } + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); +} + +/// A projection emitting the literal `value`. +fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Literal(ScalarValue::Float64(value)), + }], + child, + }) + }) +} + +/// Signed zeros and different non-finite literals are different groups; +/// JSON's `null` for non-finite floats never decides equality. +#[test] +fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 517fb9d32..49e0709e4 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -1,11 +1,12 @@ //! Memo scope: the memo, Stage 0's and Stage 1's one type, and each query //! over its groups; the operations over the whole memo -//! (concepts doc `memo-groups-properties-requirements.md`, section 3; #1203). +//! (concepts doc `memo-groups-properties-requirements.md`, section 3). use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; mod hash_consing; +use super::candidate::parameters_fit; use super::group::{ derive_group_properties, validate_expression_over_input_groups, MemoGroupAsInput, MemoGroupError, @@ -13,16 +14,16 @@ use super::group::{ use crate::accuracy::AccuracyTarget; use crate::ir::operator::input::InputOrTemporaryBuild; use crate::ir::schema::Schema; -use crate::ir::Operator; use crate::ir::QueryRoot; +use crate::ir::{NonASAPOp, Operator}; use crate::stages::{ CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, MemoGroupId, }; -use crate::workload::{DataWorkload, SourceFacts}; +use crate::workload::bounds::source_name; +use crate::workload::{series_facts, DataWorkload, SourceFacts}; use hash_consing::{ - expression_hash, same_expression_and_logical_properties, same_serialized_expression, - serialized_expression, + expression_hash, same_candidate, same_serialized_expression, serialized_expression, }; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with @@ -77,6 +78,32 @@ impl Memo { &self.groups } + /// The most series (PromQL) or groups of rows (SQL) the `Scan` groups + /// below `group` read: the sum of their sources' declared + /// `SourceFacts::series_count`; `None` when one declares none. Every + /// value of a per-series or grouped result over those scans comes from + /// at least one of them, so it bounds the values one evaluation of + /// `group` returns. Example: a readout of `quantile_over_time(0.9, + /// m[5m])` over `m` with 1000 series returns at most 1000 values, which + /// `max(...)` over it reads at once. + pub fn series_count_scanned_below(&self, group: MemoGroupId) -> Option { + let mut seen = HashSet::new(); + let mut pending = vec![group]; + let mut total: u64 = 0; + while let Some(group) = pending.pop() { + if !seen.insert(group) { + continue; + } + let expression = &self.groups[group.0 as usize].expression; + if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = expression { + let (facts, _) = series_facts(&self.source_facts, source_name(source))?; + total = total.checked_add(facts.series_count?)?; + } + pending.extend(expression.children().into_iter().copied()); + } + Some(total) + } + /// The queries, by index in the workload. pub fn queries(&self) -> &[MemoQuery] { &self.queries @@ -198,16 +225,19 @@ impl Memo { /// Adds `candidate` to `group`'s candidates and returns its id, unless /// the group already has a candidate with the same expression - /// (`same_expression`) and logical properties: then that candidate's - /// id, and `candidate` is dropped. A new candidate makes `group` a parent - /// of each of its input groups, and `group`'s queries reach each input - /// group and every group below it. + /// (`same_expression`), logical properties and input requirements: then + /// that candidate's id, and `candidate` is dropped. A new candidate makes + /// `group` a parent of each of its input groups, and `group`'s queries + /// reach each input group and every group below it. /// /// Fails, leaving the memo unchanged, when the candidate's expression /// is not valid over its input groups /// (`validate_expression_over_input_groups`), - /// or when its output schema or kind differs from the group's: then it - /// does not compute the group's result. + /// when its output schema or kind differs from the group's (then it + /// does not compute the group's result), or when its θ does not fit the + /// group's output: a candidate of a sketch state group (a build, or a + /// merge, `Filter`, `TimeRange` or `TimeShift` passing states on) has a + /// θ from that algorithm's list, any other candidate none. pub fn add_candidate( &mut self, group: MemoGroupId, @@ -216,7 +246,7 @@ impl Memo { if let Some(index_in_group) = self.groups[group.0 as usize] .candidates .iter() - .position(|known| same_expression_and_logical_properties(known, &candidate)) + .position(|known| same_candidate(known, &candidate)) { return Ok(CandidateForMemoGroupId { group, @@ -237,6 +267,11 @@ impl Memo { if schema != group_properties.schema || result_kind != group_properties.result_kind { return Err(CandidateError::OutputDiffersFromGroup { group }); } + parameters_fit( + group_properties.sketch_algorithm_of_state(), + candidate.logical_properties.parameters(), + ) + .map_err(CandidateError::LogicalProperties)?; let candidates = &mut self.groups[group.0 as usize].candidates; let input_groups: Vec = candidate .expression @@ -314,6 +349,34 @@ impl Memo { candidate.input_requirements = input_requirements; } + /// The candidates `consumer` may read on input `input`: those of the + /// input group, except that a consumer with a θ (a build, or a candidate + /// passing states on) reads only producers with the same θ or none. + pub fn producers( + &self, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Vec { + let candidate = self.candidate(consumer); + let group = *candidate.expression.operator.children()[input]; + let theta = candidate.logical_properties.parameters(); + self.groups[group.0 as usize] + .candidates + .iter() + .enumerate() + .filter( + |(_, producer)| match (theta, producer.logical_properties.parameters()) { + (Some(theta), Some(produced)) => theta == produced, + _ => true, + }, + ) + .map(|(index_in_group, _)| CandidateForMemoGroupId { + group, + index_in_group, + }) + .collect() + } + /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for /// candidate traversal, whatever order rules appended groups in. `None` @@ -355,16 +418,7 @@ impl Memo { } (order.len() == count).then_some(order) } -} - -/// Lists `parent` among `group`'s parents, once. -fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { - if !group.parents.contains(&parent) { - group.parents.push(parent); - } -} -impl Memo { /// The groups some query's root reads, ascending: the groups of the /// queries' roots that are a query's result or a scalar query's operand. pub fn root_groups(&self) -> BTreeSet { @@ -383,5 +437,12 @@ impl Memo { } } +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } +} + #[cfg(test)] mod tests; diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 136f71d57..69616a20b 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -1,29 +1,17 @@ -//! The `Memo` methods: hash-consing groups ([`Memo::find_or_add_group`]) and -//! storing candidates ([`Memo::add_candidate`]), on the concepts doc's KLL +//! The `Memo` operations other than the equality they rely on: storing +//! candidates ([`Memo::add_candidate`]), parents and queries, requirements, +//! and output schemas of pending expressions, on the concepts doc's KLL //! example (section 1: G1 the latency scan, G2 its KLL state). use super::*; -use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; -use crate::ir::operator::AggIntent; -use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; -use crate::ir::scalar::{ProjectItem, SortKey}; -use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; -use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; +use crate::ir::operator::arguments::Reduction; +use crate::ir::schema::FieldDataType; +use crate::ir::NonASAPOp; use crate::stages::logical::group::MemoGroupAsInput; -use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; +use crate::stages::LogicalPropertiesError; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; -use crate::test_support::memo_fixtures::{aggregate_with, readout, scan, state}; - -/// An unordered `LIMIT 10` over `input`, whose copies may return -/// different rows. -fn unordered_limit(input: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: input, - }) -} +use crate::test_support::memo_builders::{query, try_candidate}; +use crate::test_support::memo_fixtures::{readout, scan, state}; /// `Project(input)` keeping its first column. fn project_first_column(input: MemoGroupId) -> Operator { @@ -36,98 +24,11 @@ fn project_first_column(input: MemoGroupId) -> Operator { }) } -/// A candidate computing `operator` with `parameters`. -fn candidate( - operator: Operator, - parameters: Option, -) -> CandidateForMemoGroup { - let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; - CandidateForMemoGroup { - // Built directly: the doc example's k = 200 and k = 345 are not - // in today's KLL list, which `LogicalProperties::new` checks. - logical_properties: LogicalProperties { parameters }, - expression: CandidateExpression { operator }, - input_requirements, - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - } -} - -/// A query rooted at `group`, without an accuracy target or $n_q$. -fn query(group: MemoGroupId) -> MemoQuery { - MemoQuery { - root: QueryRoot::Operator(group), - accuracy_target: None, - number_of_values_per_query_evaluation: None, - } -} - /// KLL `k`. -fn kll_k(k: u32) -> Option { +pub(super) fn kll_k(k: u32) -> Option { Some(SketchParams::Kll { k }) } -/// An equal expression over the same input groups is found, not added. -#[test] -fn find_or_add_group_returns_the_group_of_an_equal_expression() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let g2 = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - assert_eq!( - memo.find_or_add_group(scan("latency", &[], vec![])) - .unwrap(), - g1 - ); - assert_eq!( - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(), - g2 - ); - assert_ne!( - memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), - g1 - ); - assert_eq!(memo.groups().len(), 3); - assert_eq!( - memo.groups()[g2.0 as usize].expression.children(), - vec![&g1] - ); -} - -/// The same operator over different input groups is a different group. -#[test] -fn find_or_add_group_separates_equal_operators_over_different_inputs() { - let mut memo = Memo::new(&DataWorkload::default()); - let latency = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); - assert_ne!( - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) - .unwrap(), - memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) - .unwrap() - ); -} - -/// An expression whose copies may return different rows (an unordered -/// `LIMIT`) gets a new group every time. -#[test] -fn find_or_add_group_never_returns_an_unshareable_group() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - assert_ne!( - memo.find_or_add_group(unordered_limit(g1)).unwrap(), - memo.find_or_add_group(unordered_limit(g1)).unwrap() - ); -} - /// A new group is a parent of each of its input groups, listed once. #[test] fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { @@ -144,92 +45,6 @@ fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); } -/// KLL builds with k = 200 and k = 345 are two candidates of one group -/// G2: θ is a logical property, not part of the expression. -#[test] -fn kll_builds_with_k_200_and_k_345_share_one_group() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let k200_group = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let k200 = memo - .add_candidate( - k200_group, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - let k345_group = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let k345 = memo - .add_candidate( - k345_group, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(345), - ), - ) - .unwrap(); - assert_eq!(k200_group, k345_group); - assert_ne!(k200, k345); - assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); - assert_eq!( - memo.candidate(k345).logical_properties.parameters(), - kll_k(345).as_ref() - ); -} - -/// A candidate with the same expression and logical properties as one -/// in the group is not added again; its id is returned. -#[test] -fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan("latency", &[], vec![])) - .unwrap(); - let g2 = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) - .unwrap(); - let first = memo - .add_candidate( - g2, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - let again = memo - .add_candidate( - g2, - candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(200), - ), - ) - .unwrap(); - assert_eq!(first, again); - assert_eq!( - first, - CandidateForMemoGroupId { - group: g2, - index_in_group: 0 - } - ); - let other_size = candidate( - state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), - kll_k(345), - ); - assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); - assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); -} - /// A new candidate makes its group a parent of each group it reads, and /// the group's queries reach those groups and every group below them. #[test] @@ -245,7 +60,7 @@ fn add_candidate_maintains_parents_and_queries() { let g3 = memo .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) .unwrap(); - assert_eq!(memo.add_query(query(g3)), 0); + assert_eq!(query(&mut memo, g3, None), 0); // A rule adds a state group G5 over another scan G4: no query // reaches them yet. let g4 = memo @@ -256,9 +71,11 @@ fn add_candidate_maintains_parents_and_queries() { .unwrap(); assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); // G3 gains a candidate reading G5: G5 and G4 now serve query 0. - memo.add_candidate( + try_candidate( + &mut memo, g3, - candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g5), None), + readout(SketchStatistic::Quantile { q: 0.95 }, g5), + None, ) .unwrap(); assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); @@ -268,7 +85,7 @@ fn add_candidate_maintains_parents_and_queries() { // Query 1 is G6 = Project(G3): it reaches G3 and everything below // it, through expressions and candidates, in ascending order. let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); - assert_eq!(memo.add_query(query(g6)), 1); + assert_eq!(query(&mut memo, g6, None), 1); for group in [g1, g2, g3, g4, g5] { assert_eq!( memo.groups()[group.0 as usize].reached_by_queries(), @@ -292,7 +109,7 @@ fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { .unwrap(); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate(g2, candidate(readout(SketchStatistic::Quantile { q: 0.5 }, g2), None)), + try_candidate(&mut memo, g2, readout(SketchStatistic::Quantile { q: 0.5 }, g2), None), Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 )); assert_eq!( @@ -329,9 +146,11 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { ); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate( + try_candidate( + &mut memo, g3, - candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g1), None) + readout(SketchStatistic::Quantile { q: 0.95 }, g1), + None ), Err(CandidateError::NotValidOverInputGroups(_)) )); @@ -363,224 +182,6 @@ fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); } -/// `Scan m`: `[ts, service, value, latency]`, no unique key. -fn service_scan() -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - }) -} - -/// Whether `a` and `b`, each built into one memo through -/// `find_or_add_group`, are one group. -fn one_group( - a: impl FnOnce(&mut Memo) -> MemoGroupId, - b: impl FnOnce(&mut Memo) -> MemoGroupId, -) -> bool { - let mut memo = Memo::new(&DataWorkload::default()); - let a = a(&mut memo); - let b = b(&mut memo); - a == b -} - -/// `operator` over the group of [`service_scan`]. -fn over_scan( - operator: impl FnOnce(MemoGroupId) -> Operator, -) -> impl FnOnce(&mut Memo) -> MemoGroupId { - |memo| { - let scan = memo.find_or_add_group(service_scan()).unwrap(); - memo.find_or_add_group(operator(scan)).unwrap() - } -} - -/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. -fn quantile_agg( - by: Vec, - col: Option, - q: f64, - filter: Option>, -) -> impl FnOnce(&mut Memo) -> MemoGroupId { - over_scan(move |child| { - aggregate_with( - Reduction::by(by), - vec![(AggIntent::Quantile { col, q }, filter)], - None, - child, - ) - }) -} - -/// Aggregates that differ in a column, a quantile or a filter are -/// different groups. -#[test] -fn aggregates_that_differ_in_an_argument_are_different_groups() { - assert!(!one_group( - quantile_agg(vec![1], Some(2), 0.5, None), - quantile_agg(vec![1], Some(3), 0.5, None) - )); - assert!(!one_group( - quantile_agg(vec![], Some(2), 0.5, None), - quantile_agg(vec![], Some(2), 0.9, None) - )); - let latency_above_1 = Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: ExprSemantics::Sql, - }); - assert!(!one_group( - quantile_agg(vec![1], Some(2), 0.5, None), - quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) - )); -} - -/// Equal expressions without a unique key are one group when they return -/// the same rows each time: a keyless scan, a global aggregate, a -/// `without` aggregate. -#[test] -fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { - let table_scan = |memo: &mut Memo| { - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - })) - .unwrap() - }; - assert!(one_group(table_scan, table_scan)); - assert!(one_group( - quantile_agg(vec![], Some(2), 0.9, None), - quantile_agg(vec![], Some(2), 0.9, None) - )); - let without = || { - over_scan(|child| { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having: None, - child, - }) - }) - }; - assert!(one_group(without(), without())); -} - -/// `DISTINCT` on every column returns the same rows each time and is one -/// group; a `Dedup` on some columns keeps an unspecified row per key. -#[test] -fn dedup_is_one_group_only_on_every_column() { - let dedup = |cols: Vec| { - over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) - }; - assert!(one_group(dedup(vec![]), dedup(vec![]))); - assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); -} - -/// Two unordered `LIMIT 10`s may return different rows: two groups, and -/// so are equal parents over them. -#[test] -fn unordered_limits_and_their_parents_are_different_groups() { - assert!(!one_group( - over_scan(unordered_limit), - over_scan(unordered_limit) - )); - let distinct_over_limit = |memo: &mut Memo| { - let limit = over_scan(unordered_limit)(memo); - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { - cols: vec![], - child: limit, - })) - .unwrap() - }; - assert!(!one_group(distinct_over_limit, distinct_over_limit)); -} - -/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows -/// of a fixed order (one group); over a `Sort` partitioned differently, -/// which rows come first depends on input order (two groups). -#[test] -fn limits_over_a_sort_with_their_partitions_are_one_group() { - let limit = |sort_partitions: GroupKeys| { - move |memo: &mut Memo| { - let sorted = over_scan(|child| { - Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child, - }) - })(memo); - memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: sorted, - })) - .unwrap() - } - }; - assert!(one_group( - limit(GroupKeys::default()), - limit(GroupKeys::default()) - )); - assert!(!one_group( - limit(GroupKeys::by(vec![1])), - limit(GroupKeys::by(vec![1])) - )); -} - -/// A projection emitting the literal `value`. -fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { - over_scan(move |child| { - Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::Literal(ScalarValue::Float64(value)), - }], - child, - }) - }) -} - -/// Signed zeros and different non-finite literals are different groups; -/// JSON's `null` for non-finite floats never decides equality. -#[test] -fn signed_zero_and_non_finite_literals_are_different_groups() { - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); - } - assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); -} - /// A KLL state over the latency scan and its p99 readout, both pending: the /// readout's schema is the one its group gets once both are added, and /// nothing is added while deriving it. @@ -625,3 +226,188 @@ fn output_schema_of_a_pending_expression_rejects_what_find_or_add_group_rejects( assert!(memo.output_schema_of_pending_expression(&[], &bad).is_err()); assert!(memo.find_or_add_group(bad).is_err()); } + +/// One readout's required accuracy. +fn required(max_error: f64) -> Vec { + vec![LogicalRequirements { + required_accuracy: Some(crate::stages::RequiredAccuracy { + metric: crate::accuracy::AccuracyMetric::Rank, + max_error, + failure_probability: 1.0, + }), + }] +} + +/// `Memo::set_candidate_input_requirements` +/// replaces $r^{\log}$ of the one candidate it names; two readouts of one +/// state with different required accuracy are two candidates. +#[test] +fn set_candidate_input_requirements_replaces_only_the_named_candidate() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let p90 = || readout(SketchStatistic::Quantile { q: 0.9 }, g2); + let g3 = memo.find_or_add_group(p90()).unwrap(); + let first = try_candidate(&mut memo, g3, p90(), None).unwrap(); + memo.set_candidate_input_requirements(first, required(0.01)); + let second = try_candidate(&mut memo, g3, p90(), None).unwrap(); + assert_ne!(first, second, "an unset copy is a new candidate"); + memo.set_candidate_input_requirements(second, required(0.02)); + let requirements: Vec<_> = memo.groups()[g3.0 as usize] + .candidates + .iter() + .map(|c| c.input_requirements.clone()) + .collect(); + assert_eq!(requirements, vec![required(0.01), required(0.02)]); +} + +/// A merge of KLL states carries the θ of the +/// states it reads. Without one it is rejected; `merge[k=547]` reads only +/// the k = 547 build of its input group. +#[test] +fn a_merge_carries_the_parameters_of_the_states_it_reads() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll_state = || state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1); + let g2 = memo.find_or_add_group(kll_state()).unwrap(); + for k in [269, 547] { + try_candidate(&mut memo, g2, kll_state(), kll_k(k)).unwrap(); + } + let merge = Operator::ASAP(crate::ir::ASAPOp::SummaryMerge { + children: vec![g2], + group_by: None, + }); + let g3 = memo.find_or_add_group(merge.clone()).unwrap(); + assert!(matches!( + try_candidate(&mut memo, g3, merge.clone(), None), + Err(CandidateError::LogicalProperties( + LogicalPropertiesError::SketchBuildWithoutParameters + )) + )); + let merged = try_candidate(&mut memo, g3, merge, kll_k(547)).unwrap(); + assert_eq!( + memo.producers(merged, 0), + vec![CandidateForMemoGroupId { + group: g2, + index_in_group: 1 + }] + ); +} + +/// One requirement per input group, or a panic. +#[test] +#[should_panic(expected = "one input requirement per input group")] +fn set_candidate_input_requirements_panics_on_a_length_mismatch() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll_state = || state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1); + let g2 = memo.find_or_add_group(kll_state()).unwrap(); + let build = try_candidate(&mut memo, g2, kll_state(), kll_k(269)).unwrap(); + memo.set_candidate_input_requirements(build, Vec::new()); +} + +/// `add_candidate` checks θ against the group's output, for every operator +/// kind: a sketch state's candidates (a build, a chain, a merge) need θ of +/// its algorithm from its list; any other candidate (an exact build, a +/// readout) has none. No Hydra θ is listed yet, so no Hydra build is valid. +#[test] +fn add_candidate_checks_parameters_against_the_groups_output() { + use crate::ir::operator::arguments::Reduction; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::ASAPOp; + use crate::summary_type::{ExactKind, HydraParams, SummaryUpdate}; + use LogicalPropertiesError as Invalid; + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let kll_state = memo.find_or_add_group(state(kll.clone(), g1)).unwrap(); + let kll_chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: g1, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![kll_state], + group_by: None, + }); + // k = 270 is KLL's structure but not in its list. + let parameters = [ + None, + kll_k(269), + Some(SketchParams::Cms { + width: 272, + depth: 5, + }), + Some(SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 1_024, + })), + kll_k(270), + ]; + let no_parameters = Err(Invalid::ParametersOnExactOperator); + let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); + let not_in_list = Err(Invalid::ParametersNotInList); + let exact = [ + Ok(()), + no_parameters, + no_parameters, + no_parameters, + no_parameters, + ]; + // Another algorithm's parameters are not in KLL's list either. + let kll_state_output = [ + needs_parameters, + Ok(()), + not_in_list, + not_in_list, + not_in_list, + ]; + // Per operator: the result for each of `parameters`. + let table = [ + (state(kll, g1), kll_state_output), + ( + state(FieldDataType::Sketch(SketchAlgorithm::HydraCms), g1), + [ + needs_parameters, + not_in_list, + not_in_list, + not_in_list, + not_in_list, + ], + ), + (state(FieldDataType::exact(ExactKind::Sum), g1), exact), + (kll_chain, kll_state_output), + ( + readout(SketchStatistic::Quantile { q: 0.5 }, kll_state), + exact, + ), + (merge, kll_state_output), + ]; + for (operator, by_parameters) in table { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + for (parameters, expected) in parameters.iter().zip(by_parameters) { + let added = try_candidate(&mut memo, group, operator.clone(), parameters.clone()); + let added = match added { + Ok(_) => Ok(()), + Err(CandidateError::LogicalProperties(error)) => Err(error), + Err(error) => panic!("{error}"), + }; + assert_eq!(added, expected, "{:?} {parameters:?}", operator.kind_name()); + } + } +} diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 56f1d9232..16125313f 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -2,7 +2,9 @@ //! rule, the logical edges and the candidates dead-candidate elimination //! removed. -use crate::stages::{EvaluationGrid, LogicalEdge, Memo, PrunedLogicalCandidate}; +use std::collections::BTreeSet; + +use crate::stages::{EvaluationGrid, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate}; /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] @@ -18,8 +20,24 @@ pub struct LogicalASAPMemoCandidates { /// Per group, by `MemoGroupId`: the grid a result of the group is /// evaluated on, the coarsest grid holding every evaluation time of the /// queries reaching it (X = `sum_over_time(m[5m])` read every 60 s and - /// every 120 s: 60 s); `None` when those times are not on one known - /// grid. A result of the group materialized across evaluations is + /// every 120 s: 60 s; below a PromQL subquery `[10m:30s]`, its 30 s + /// steps); `None` when those times are not on one known grid (below a + /// subquery without a resolution). A result of the group materialized across evaluations is /// stored at these times. pub evaluation_grid_of_group: Vec>, + /// Indexed by query: the groups this query reaches through a consumer with no + /// error contract for its metric; in these groups the query may only use exact candidates. + /// Q0 = `quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, + /// m[5m])` and Q1 = the same ratio `> 1`, both relative 0.05: Q0 may + /// read the DDSketch readouts, while Q1's comparison has no contract, + /// so its set holds the ratio group and every group below it. (Both + /// list the rows below a sketch build, which has no contract.) A query + /// whose whole-result failure probability has no per-value one ($n_q$ + /// unknown) also lists each readout group another query gave a level, + /// with the groups below it. A group is listed for a query, not + /// pruned: other queries may still read its approximate candidates, so + /// Stage 3 must enforce it per query, in each query's selection + /// context; Pass 2 only lists it. Empty for a query without an accuracy + /// target. + pub exact_only_groups_of_query: Vec>, } diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index b75e6594d..b546a477c 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -1,28 +1,31 @@ -//! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group -//! → candidate → candidate input → edge → Stage 1's output). +//! The logical side: Stage 0's and Stage 1's memo, by scope. +//! +//! - `memo` — the memo and its queries. +//! - `group` — a memo group and the properties its candidates share +//! (column lineage and sign, aggregate coverage). +//! - `candidate` — a group's candidates. +//! - `requirements` — what a candidate requires of each input. +//! - `edge` — the logical edges between candidates. +//! - `memo_candidates` — Stage 1's output. -mod aggregate_coverage; mod candidate; -mod column_properties; mod edge; mod group; mod memo; mod memo_candidates; mod requirements; -mod view_coverage; -pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; pub use candidate::{ - CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, - LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, + listed_build_parameters, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, + OriginOfCandidateForMemoGroup, }; -pub use column_properties::{ColumnLineage, ValueSign}; pub use edge::LogicalEdge; pub use group::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, GroupProperties, MemoGroup, - MemoGroupError, MemoGroupId, + ranked_column_of_group, raw_data_samples_of_metric_of_group, AggregateCoverage, + AggregateDefinition, ColumnLineage, CoverageError, GroupProperties, MemoGroup, MemoGroupError, + MemoGroupId, ValueSign, }; pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; -pub use view_coverage::view_coverage; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index aafdbf2ab..ba1accd63 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -1,20 +1,22 @@ -//! The Stage 0 → Stage 1 boundary: the workload's memo groups, without -//! candidates ([`Memo`]). Inside Stage 1, Pass 1 → Pass 2: the memo with -//! its Pass 1 candidates. The Stage 1 → Stage 2 boundary: the memo of logical groups and their -//! implementation candidates that Stage 1 (Pass 1 and Pass 2) produces and -//! Stage 2 implements (#1047, #1054). The Stage 2 → Stage 3 boundary: the -//! physical candidates Stage 2 builds and Stage 3 selects from. Stage -//! crates never depend on each other, so the types more than one stage reads -//! live here. +//! The types one planning stage hands the next, so that stage crates never +//! depend on each other: +//! +//! - `logical` — Stage 0's and Stage 1's memo ([`Memo`]: groups, queries, +//! candidates and their edges), and Stage 1's output +//! ([`LogicalASAPMemoCandidates`]) that Stage 2 implements. +//! - `physical` — Stage 2's output ([`PhysicalASAPMemoCandidates`]): the +//! physical candidates and edges Stage 3 selects from. +//! - `pruning` — what each stage removed and why ([`PrunedCandidate`], +//! [`PruneReason`]), one shape for every stage. mod logical; mod physical; mod pruning; pub use logical::{ - ranked_column_of_group, raw_data_samples_of_metric_of_group, view_coverage, AggregateCoverage, - AggregateDefinition, CandidateError, CandidateExpression, CandidateForMemoGroup, - CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, + listed_build_parameters, ranked_column_of_group, raw_data_samples_of_metric_of_group, + AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index c5891c112..92bd5f997 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -1,5 +1,5 @@ //! Candidate scope: each logical candidate's physical alternatives, one per -//! valid $p^{\mathrm{phys}} = (\tau, \ell)$ (#1200). +//! valid $p^{\mathrm{phys}} = (\tau, \ell)$. use crate::ir::properties::ExecutionTiming; use crate::ir::{ASAPOp, Operator}; diff --git a/crates/types/src/stages/physical/edge.rs b/crates/types/src/stages/physical/edge.rs index 7df48ad9a..07f30e68b 100644 --- a/crates/types/src/stages/physical/edge.rs +++ b/crates/types/src/stages/physical/edge.rs @@ -1,5 +1,5 @@ //! Edge scope: the physical edges $\mathcal E$ with $\alpha^{\log}$ and -//! $\alpha^{\mathrm{phys}}$ (#1201). +//! $\alpha^{\mathrm{phys}}$. use crate::stages::PhysicalCandidateId; diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs deleted file mode 100644 index 502592071..000000000 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ /dev/null @@ -1,421 +0,0 @@ -//! Stage 2's output: each logical candidate's physical alternatives -//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and -//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and -//! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; -//! Stage 2 prices nothing. - -use std::collections::HashMap; - -use crate::ir::QueryRoot; -use crate::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, PrunedCandidate, -}; - -/// Stage 2's output, which Stage 3 selects from. -#[derive(Debug, Clone)] -pub struct PhysicalASAPMemoCandidates { - /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, - /// $r^{\log}_{c,i}$, and the logical edges. - pub logical_candidates: LogicalASAPMemoCandidates, - /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. - pub physical_candidates: Vec, - /// The alternatives Stage 2 pruned, in pruning order, each as the - /// [`PhysicalCandidate`] it would have been: candidates are numbered - /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. - pub pruned_candidates: Vec>, - /// $\mathcal E$, grouped by (consumer, input); indexed by - /// [`PhysicalEdgeId`]. - edges: Vec, - /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). - admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, - /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. - admissible_edges_out_of_candidate: Vec>, -} - -impl PhysicalASAPMemoCandidates { - /// Stage 2's output over `physical_candidates` and `edges` (grouped by - /// (consumer, input)), with the admissible-edge indexes built once. - /// Query roots are read from `logical_candidates`, not copied. - pub fn new( - logical_candidates: LogicalASAPMemoCandidates, - physical_candidates: Vec, - edges: Vec, - pruned_candidates: Vec>, - ) -> Self { - let _ = ( - logical_candidates, - physical_candidates, - edges, - pruned_candidates, - ); - todo!("#1207 PR 5: index the admissible edges") - } - - /// $\mathcal E$: every edge, in (consumer, input) order. - pub fn edges(&self) -> impl Iterator + '_ { - self.edges - .iter() - .enumerate() - .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) - } - - /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input - /// `input_index_of_consumer` of `consumer` - /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). - pub fn admissible_edges_into( - &self, - consumer: PhysicalCandidateId, - input_index_of_consumer: usize, - ) -> &[PhysicalEdgeId] { - let _ = ( - consumer, - input_index_of_consumer, - &self.admissible_edges_into_input, - ); - todo!("#1207 PR 5") - } - - /// The admissible edges out of `producer`: every consumer that may share it. - pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { - let _ = (producer, &self.admissible_edges_out_of_candidate); - todo!("#1207 PR 5") - } -} - -/// The output references of `query_roots`: each operator reference, in -/// order (a scalar root's in `map_operator_refs` order), with the index of -/// the query whose root it is. -pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { - let mut references = Vec::new(); - for (query, root) in query_roots.iter().enumerate() { - match root { - QueryRoot::Operator(reference) => references.push((query, reference.clone())), - QueryRoot::Scalar(expression) => { - expression.map_operator_refs(&mut |reference| { - references.push((query, reference.clone())) - }); - } - } - } - references -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::properties::ExecutionTiming; - use crate::ir::{ASAPOp, Operator}; - use crate::stages::{ - producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, - Lifetime, PhysicalProperties, - }; - use crate::workload::DataWorkload; - - use crate::plan::materialized_view::RefreshWindow; - - use crate::ir::operator::arguments::{Reduction, Source}; - - use crate::ir::scalar::expr_ir::ColumnRef; - - use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; - use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; - - use crate::ir::{NonASAPOp, TimeRangeKind}; - - use crate::stages::StorageMedium; - - use crate::stages::physical::execution_context::EvaluationGrid; - - use crate::stages::{ - CandidateExpression, CandidateForMemoGroup, LogicalASAPMemoCandidates, LogicalProperties, - LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, - }; - - use ExecutionTiming::{IngestionTime, QueryTime}; - - /// The KLL build over the scan: not a chain. - const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(1), - index_in_group: 0, - }; - - /// The chain of tumbling windows 5 s wide over the scan. - const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(2), - index_in_group: 0, - }; - - /// The chain of tumbling windows 10 s wide over the scan. - const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(3), - index_in_group: 0, - }; - - /// X = the scan's last 5 minutes (`m[5m]`): a result that depends on - /// the evaluation time and is not a chain. - const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { - group: MemoGroupId(4), - index_in_group: 0, - }; - - /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains - /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last - /// 5 minutes, each with its own expression as its one candidate. Built - /// through `find_or_add_group` and `add_candidate`. - fn memo() -> Memo { - let mut memo = Memo::new(&DataWorkload::default()); - let scan = memo - .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); - let build = Operator::ASAP(ASAPOp::SummaryAgg { - child: scan, - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }); - let chain = |tumbling_window_width_ms| { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: scan, - summary_type: kll.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - tumbling_window_width_ms, - }) - }; - for operator in [build, chain(5_000), chain(10_000)] { - let group = memo.find_or_add_group(operator.clone()).unwrap(); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new( - &expression, - Some(SketchParams::Kll { k: 269 }), - ) - .unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; - memo.add_candidate(group, candidate).unwrap(); - } - let operator = Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - }); - let group = memo.find_or_add_group(operator.clone()).unwrap(); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; - memo.add_candidate(group, candidate).unwrap(); - memo - } - - /// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; - /// `delivers_on` reads nothing else. - fn candidates( - memo: Memo, - evaluation_grid_of_group: Vec>, - ) -> PhysicalASAPMemoCandidates { - PhysicalASAPMemoCandidates { - logical_candidates: LogicalASAPMemoCandidates { - memo, - edges: vec![], - pruned: vec![], - evaluation_grid_of_group, - }, - physical_candidates: vec![], - pruned_candidates: vec![], - edges: vec![], - admissible_edges_into_input: HashMap::new(), - admissible_edges_out_of_candidate: vec![], - } - } - - fn candidate( - logical_candidate_id: CandidateForMemoGroupId, - timing: ExecutionTiming, - lifetime: Lifetime, - ) -> PhysicalCandidate { - PhysicalCandidate { - logical_candidate_id, - physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), - } - } - - fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { - ExecutionContext::Evaluation { - grid: EvaluationGrid { - period_ms, - phase_ms, - }, - } - } - - fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { - ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) - } - - const MEMORY: StorageMedium = StorageMedium::Memory; - - /// A `None` producer runs in the consumer's context and delivers on - /// every one; a `OneEvaluation` producer only on an evaluation context, - /// the one evaluation its result belongs to. Neither reads the memo. - #[test] - fn delivers_on_by_lifetime_without_windows() { - let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); - let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); - let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); - let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); - for (producer, context, delivers) in [ - (&unmaterialized, evaluation(7_000, 3_000), true), - (&unmaterialized, ingestion(5_000, 0), true), - (&unmaterialized, materialized, true), - (&one_evaluation, evaluation(7_000, 3_000), true), - (&one_evaluation, ingestion(5_000, 0), false), - (&one_evaluation, materialized, false), - ] { - assert_eq!( - producer.delivers_on(&candidates, &context), - delivers, - "{:?} in {context:?}", - producer.physical_properties - ); - } - } - - /// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase - /// 0) delivers when $w$ divides the context's period (or refresh-window - /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s - /// serves both. - #[test] - fn delivers_on_a_chain_when_its_width_divides_the_context_times() { - let candidates = candidates(memo(), vec![None; 5]); - let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); - let ingestion_10s = candidate( - WINDOWS_10S, - IngestionTime, - Lifetime::AcrossEvaluations(MEMORY), - ); - for (producer, context, delivers) in [ - // Evaluation grids: w divides period and phase. - (&across_5s, evaluation(10_000, 0), true), - (&across_5s, evaluation(10_000, 5_000), true), - (&across_5s, evaluation(10_000, 2_000), false), - (&across_5s, evaluation(7_000, 0), false), - (&ingestion_10s, evaluation(10_000, 0), true), - (&ingestion_10s, evaluation(10_000, 5_000), false), - // Refresh windows: w divides width and phase. - (&across_5s, ingestion(5_000, 0), true), - (&across_5s, ingestion(10_000, 5_000), true), - (&across_5s, ingestion(10_000, 2_500), false), - (&ingestion_10s, ingestion(5_000, 0), false), - ] { - assert_eq!( - producer.delivers_on(&candidates, &context), - delivers, - "{:?} {:?} in {context:?}", - producer.logical_candidate_id, - producer.physical_properties - ); - } - } - - /// A stored result that is not a chain delivers when its group's - /// evaluation grid holds every time of the context: X = - /// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is - /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or - /// another phase. Without a known grid it serves none. - #[test] - fn delivers_on_a_stored_result_on_its_group_grid() { - let grid_60s = Some(EvaluationGrid { - period_ms: 60_000, - phase_ms: 0, - }); - let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); - let without_grid = candidates(memo(), vec![None; 5]); - let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); - for (candidates, context, delivers) in [ - (&on_grid, evaluation(60_000, 0), true), - (&on_grid, evaluation(120_000, 0), true), - (&on_grid, evaluation(120_000, 60_000), true), - (&on_grid, evaluation(30_000, 0), false), - (&on_grid, evaluation(60_000, 15_000), false), - (&on_grid, ingestion(60_000, 0), true), - (&without_grid, evaluation(60_000, 0), false), - ] { - assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); - } - } - - /// `producer_satisfies_physical_requirements` over every valid producer state and every valid - /// consumer state: a query-time producer serves only query-time - /// consumers, an ingestion-time producer that is not materialized only - /// ingestion-time consumers, and one materialized across evaluations - /// serves both. Chain widths do not matter: alignment is Stage 3's - /// per-context check. - #[test] - fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { - let states = |medium| { - [ - candidate(BUILD, QueryTime, Lifetime::None), - candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), - candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), - candidate(BUILD, IngestionTime, Lifetime::None), - candidate( - WINDOWS_5S, - IngestionTime, - Lifetime::AcrossEvaluations(medium), - ), - ] - }; - // Rows: producer state; columns: consumer state, in `states` order. - let expected = [ - [true, true, true, false, false], - [true, true, true, false, false], - [true, true, true, false, false], - [false, false, false, true, true], - [true, true, true, true, true], - ]; - for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { - for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { - for (producer, row) in states(producer_medium).iter().zip(expected) { - for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { - assert_eq!( - producer_satisfies_physical_requirements( - producer, - consumer.physical_properties.timing() - ), - satisfies, - "{:?} -> {:?}", - producer.physical_properties, - consumer.physical_properties - ); - } - } - } - } - } -} diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs new file mode 100644 index 000000000..523021a54 --- /dev/null +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -0,0 +1,105 @@ +//! Stage 2's output: each logical candidate's physical alternatives +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and +//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$. Stage 3 selects among them; +//! Stage 2 prices nothing. + +use std::collections::HashMap; + +use crate::ir::QueryRoot; +use crate::stages::{ + LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + PhysicalEdgeId, PrunedCandidate, +}; + +/// Stage 2's output, which Stage 3 selects from. +#[derive(Debug, Clone)] +pub struct PhysicalASAPMemoCandidates { + /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, + /// $r^{\log}_{c,i}$, and the logical edges. + pub logical_candidates: LogicalASAPMemoCandidates, + /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. + pub physical_candidates: Vec, + /// The alternatives Stage 2 pruned, in pruning order, each as the + /// [`PhysicalCandidate`] it would have been: candidates are numbered + /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. + pub pruned_candidates: Vec>, + /// $\mathcal E$, grouped by (consumer, input); indexed by + /// [`PhysicalEdgeId`]. + edges: Vec, + /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). + admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, + /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. + admissible_edges_out_of_candidate: Vec>, +} + +impl PhysicalASAPMemoCandidates { + /// Stage 2's output over `physical_candidates` and `edges` (grouped by + /// (consumer, input)), with the admissible-edge indexes built once. + /// Query roots are read from `logical_candidates`, not copied. + pub fn new( + logical_candidates: LogicalASAPMemoCandidates, + physical_candidates: Vec, + edges: Vec, + pruned_candidates: Vec>, + ) -> Self { + let _ = ( + logical_candidates, + physical_candidates, + edges, + pruned_candidates, + ); + todo!("#1207 PR 5: index the admissible edges") + } + + /// $\mathcal E$: every edge, in (consumer, input) order. + pub fn edges(&self) -> impl Iterator + '_ { + self.edges + .iter() + .enumerate() + .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) + } + + /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input + /// `input_index_of_consumer` of `consumer` + /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). + pub fn admissible_edges_into( + &self, + consumer: PhysicalCandidateId, + input_index_of_consumer: usize, + ) -> &[PhysicalEdgeId] { + let _ = ( + consumer, + input_index_of_consumer, + &self.admissible_edges_into_input, + ); + todo!("#1207 PR 5") + } + + /// The admissible edges out of `producer`: every consumer that may share it. + pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { + let _ = (producer, &self.admissible_edges_out_of_candidate); + todo!("#1207 PR 5") + } +} + +/// The output references of `query_roots`: each operator reference, in +/// order (a scalar root's in `map_operator_refs` order), with the index of +/// the query whose root it is. +pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { + let mut references = Vec::new(); + for (query, root) in query_roots.iter().enumerate() { + match root { + QueryRoot::Operator(reference) => references.push((query, reference.clone())), + QueryRoot::Scalar(expression) => { + expression.map_operator_refs(&mut |reference| { + references.push((query, reference.clone())) + }); + } + } + } + references +} + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs new file mode 100644 index 000000000..723104d35 --- /dev/null +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -0,0 +1,304 @@ +use super::*; +use crate::ir::properties::ExecutionTiming; +use crate::ir::{ASAPOp, Operator}; +use crate::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, Lifetime, + PhysicalProperties, +}; +use crate::workload::DataWorkload; + +use crate::plan::materialized_view::RefreshWindow; + +use crate::ir::operator::arguments::{Reduction, Source}; + +use crate::ir::scalar::expr_ir::ColumnRef; + +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; + +use crate::ir::{NonASAPOp, TimeRangeKind}; + +use crate::stages::StorageMedium; + +use crate::stages::physical::execution_context::EvaluationGrid; + +use crate::stages::{ + CandidateForMemoGroup, LogicalASAPMemoCandidates, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; + +use ExecutionTiming::{IngestionTime, QueryTime}; + +/// The KLL build over the scan: not a chain. +const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(1), + index_in_group: 0, +}; + +/// The chain of tumbling windows 5 s wide over the scan. +const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(2), + index_in_group: 0, +}; + +/// The chain of tumbling windows 10 s wide over the scan. +const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(3), + index_in_group: 0, +}; + +/// X = the scan's last 5 minutes (`m[5m]`): a result that depends on +/// the evaluation time and is not a chain. +const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(4), + index_in_group: 0, +}; + +/// Group 1, KLL states over the scan group 0, groups 2 and 3, chains +/// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last +/// 5 minutes, each with its own expression as its one candidate. Built +/// through `find_or_add_group` and `add_candidate`. +fn memo() -> Memo { + let mut memo = Memo::new(&DataWorkload::default()); + let scan = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let chain = |tumbling_window_width_ms| { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms, + }) + }; + for operator in [build, chain(5_000), chain(10_000)] { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = CandidateForMemoGroup::new( + operator, + Some(SketchParams::Kll { k: 269 }), + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); + memo.add_candidate(group, candidate).unwrap(); + } + let operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = + CandidateForMemoGroup::new(operator, None, OriginOfCandidateForMemoGroup::Pass1, vec![]); + memo.add_candidate(group, candidate).unwrap(); + memo +} + +/// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; +/// `delivers_on` reads nothing else. +fn candidates( + memo: Memo, + evaluation_grid_of_group: Vec>, +) -> PhysicalASAPMemoCandidates { + PhysicalASAPMemoCandidates { + logical_candidates: LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group, + exact_only_groups_of_query: vec![], + }, + physical_candidates: vec![], + pruned_candidates: vec![], + edges: vec![], + admissible_edges_into_input: HashMap::new(), + admissible_edges_out_of_candidate: vec![], + } +} + +fn candidate( + logical_candidate_id: CandidateForMemoGroupId, + timing: ExecutionTiming, + lifetime: Lifetime, +) -> PhysicalCandidate { + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } +} + +fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms, + }, + } +} + +fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) +} + +const MEMORY: StorageMedium = StorageMedium::Memory; + +/// A `None` producer runs in the consumer's context and delivers on +/// every one; a `OneEvaluation` producer only on an evaluation context, +/// the one evaluation its result belongs to. Neither reads the memo. +#[test] +fn delivers_on_by_lifetime_without_windows() { + let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); + let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); + let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); + let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); + for (producer, context, delivers) in [ + (&unmaterialized, evaluation(7_000, 3_000), true), + (&unmaterialized, ingestion(5_000, 0), true), + (&unmaterialized, materialized, true), + (&one_evaluation, evaluation(7_000, 3_000), true), + (&one_evaluation, ingestion(5_000, 0), false), + (&one_evaluation, materialized, false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.physical_properties + ); + } +} + +/// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase +/// 0) delivers when $w$ divides the context's period (or refresh-window +/// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s +/// serves both. +#[test] +fn delivers_on_a_chain_when_its_width_divides_the_context_times() { + let candidates = candidates(memo(), vec![None; 5]); + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let ingestion_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for (producer, context, delivers) in [ + // Evaluation grids: w divides period and phase. + (&across_5s, evaluation(10_000, 0), true), + (&across_5s, evaluation(10_000, 5_000), true), + (&across_5s, evaluation(10_000, 2_000), false), + (&across_5s, evaluation(7_000, 0), false), + (&ingestion_10s, evaluation(10_000, 0), true), + (&ingestion_10s, evaluation(10_000, 5_000), false), + // Refresh windows: w divides width and phase. + (&across_5s, ingestion(5_000, 0), true), + (&across_5s, ingestion(10_000, 5_000), true), + (&across_5s, ingestion(10_000, 2_500), false), + (&ingestion_10s, ingestion(5_000, 0), false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} {:?} in {context:?}", + producer.logical_candidate_id, + producer.physical_properties + ); + } +} + +/// A stored result that is not a chain delivers when its group's +/// evaluation grid holds every time of the context: X = +/// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is +/// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or +/// another phase. Without a known grid it serves none. +#[test] +fn delivers_on_a_stored_result_on_its_group_grid() { + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); + let without_grid = candidates(memo(), vec![None; 5]); + let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + for (candidates, context, delivers) in [ + (&on_grid, evaluation(60_000, 0), true), + (&on_grid, evaluation(120_000, 0), true), + (&on_grid, evaluation(120_000, 60_000), true), + (&on_grid, evaluation(30_000, 0), false), + (&on_grid, evaluation(60_000, 15_000), false), + (&on_grid, ingestion(60_000, 0), true), + (&without_grid, evaluation(60_000, 0), false), + ] { + assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + } +} + +/// `producer_satisfies_physical_requirements` over every valid producer state and every valid +/// consumer state: a query-time producer serves only query-time +/// consumers, an ingestion-time producer that is not materialized only +/// ingestion-time consumers, and one materialized across evaluations +/// serves both. Chain widths do not matter: alignment is Stage 3's +/// per-context check. +#[test] +fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { + let states = |medium| { + [ + candidate(BUILD, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), + candidate(BUILD, IngestionTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(medium), + ), + ] + }; + // Rows: producer state; columns: consumer state, in `states` order. + let expected = [ + [true, true, true, false, false], + [true, true, true, false, false], + [true, true, true, false, false], + [false, false, false, true, true], + [true, true, true, true, true], + ]; + for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (producer, row) in states(producer_medium).iter().zip(expected) { + for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { + assert_eq!( + producer_satisfies_physical_requirements( + producer, + consumer.physical_properties.timing() + ), + satisfies, + "{:?} -> {:?}", + producer.physical_properties, + consumer.physical_properties + ); + } + } + } + } +} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index b733ee832..2b3728d4d 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,6 +1,12 @@ //! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 -//! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201), -//! by scope (execution context → candidate → edge → Stage 2's output). +//! builds from the memo and Stage 3 selects from, by scope (execution +//! context → candidate → edge → Stage 2's output). +//! +//! - `execution_context` — the contexts a candidate runs in. +//! - `candidate` — a logical candidate's physical alternatives. +//! - `properties` — a candidate's physical properties (timing, lifetime). +//! - `edge` — the physical edges between candidates. +//! - `memo_candidates` — Stage 2's output. mod candidate; mod edge; diff --git a/crates/types/src/stages/physical/properties.rs b/crates/types/src/stages/physical/properties/mod.rs similarity index 67% rename from crates/types/src/stages/physical/properties.rs rename to crates/types/src/stages/physical/properties/mod.rs index 3cd8f2a40..e37dad27d 100644 --- a/crates/types/src/stages/physical/properties.rs +++ b/crates/types/src/stages/physical/properties/mod.rs @@ -1,5 +1,5 @@ //! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell)$ -//! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 +//! (concepts doc `memo-groups-properties-requirements.md`, sections 1 //! and 3): when it runs ([`ExecutionTiming`]) and how long its output lives //! ([`Lifetime`]). There is no schedule property: a `None` or `OneEvaluation` //! output runs on the schedule of the context κ that runs it, and a tumbling @@ -89,39 +89,4 @@ impl PhysicalProperties { } #[cfg(test)] -mod tests { - use super::*; - use ExecutionTiming::{IngestionTime, QueryTime}; - - /// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for - /// both media, and keeps what it was given. - #[test] - fn new_over_every_timing_and_lifetime() { - for medium in [StorageMedium::Memory, StorageMedium::Disk] { - for (timing, lifetime, valid) in [ - (QueryTime, Lifetime::None, true), - (QueryTime, Lifetime::OneEvaluation(medium), true), - (QueryTime, Lifetime::AcrossEvaluations(medium), true), - (IngestionTime, Lifetime::None, true), - (IngestionTime, Lifetime::OneEvaluation(medium), false), - (IngestionTime, Lifetime::AcrossEvaluations(medium), true), - ] { - let properties = PhysicalProperties::new(timing, lifetime); - match valid { - true => { - let properties = properties.unwrap(); - assert_eq!( - (properties.timing(), properties.lifetime()), - (timing, lifetime) - ); - assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); - } - false => assert_eq!( - properties, - Err(PhysicalPropertiesError::OneEvaluationAtIngestion) - ), - } - } - } - } -} +mod tests; diff --git a/crates/types/src/stages/physical/properties/tests.rs b/crates/types/src/stages/physical/properties/tests.rs new file mode 100644 index 000000000..5a0ad7f10 --- /dev/null +++ b/crates/types/src/stages/physical/properties/tests.rs @@ -0,0 +1,34 @@ +use super::*; +use ExecutionTiming::{IngestionTime, QueryTime}; + +/// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for +/// both media, and keeps what it was given. +#[test] +fn new_over_every_timing_and_lifetime() { + for medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (timing, lifetime, valid) in [ + (QueryTime, Lifetime::None, true), + (QueryTime, Lifetime::OneEvaluation(medium), true), + (QueryTime, Lifetime::AcrossEvaluations(medium), true), + (IngestionTime, Lifetime::None, true), + (IngestionTime, Lifetime::OneEvaluation(medium), false), + (IngestionTime, Lifetime::AcrossEvaluations(medium), true), + ] { + let properties = PhysicalProperties::new(timing, lifetime); + match valid { + true => { + let properties = properties.unwrap(); + assert_eq!( + (properties.timing(), properties.lifetime()), + (timing, lifetime) + ); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + false => assert_eq!( + properties, + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ), + } + } + } +} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index c389921c2..6a8fdeef4 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,13 +1,14 @@ -//! What a stage removed, and why: one shape for every stage (#1047 P6). -//! Each stage removes only what its own knowledge rules out (option C): -//! Stage 2 what cannot be built, run or read by a query output. Stage 0 fails with an -//! error, Pass 1 records nothing it leaves out (#1205 will show it), and -//! Pass 2 records its eliminated candidates with their own causes -//! (`PrunedLogicalCandidate`). Stage 3 prunes before -//! its selection MILP (dominated and uncostable candidates); its -//! rejections of whole plans keep their own cause (`RejectionCause` in plan -//! selection). The planner does not check whether an executor implements an -//! operator or summary type, so no reason names it. +//! What a stage removed, and why: one shape for every stage. Each stage +//! removes only what its own knowledge rules out. Pass 2 records its +//! eliminated logical candidates with their causes ([`PrunedLogicalCandidate`], +//! [`PruneCause`], [`RootPruneReason`]); Stage 2 records the physical +//! candidates that cannot be built, run or read by a query output +//! ([`PrunedCandidate`], [`PruneReason`]); Stage 3 prunes dominated and +//! uncostable candidates before its selection, and its rejections of whole +//! plans keep their own cause (`RejectionCause` in plan selection). Stage 0 +//! fails with an error and Pass 1 records nothing it leaves out. The planner +//! does not check whether an executor implements an operator or summary +//! type, so no reason names it. use crate::accuracy::AccuracyMetric; use crate::stages::{CandidateForMemoGroupId, PhysicalCandidateId, RequiredAccuracy}; @@ -22,12 +23,13 @@ pub struct PrunedLogicalCandidate { pub causes: Vec, } -/// One path from the candidate, through consumers that are themselves +/// One path from the candidate, through candidates that are themselves /// eliminated, to the edge that fails. One shortest path per failing edge. #[derive(Debug, Clone)] pub struct PruneCause { - /// The eliminated consumers on the path, nearest first. - pub through_pruned_consumers: Vec, + /// The eliminated candidates on the path, nearest first: consumers + /// above, or, for a candidate cut off from an input, producers below. + pub through_pruned_candidates: Vec, pub root: RootPruneReason, } @@ -42,12 +44,52 @@ pub enum RootPruneReason { input: usize, required: RequiredAccuracy, }, - /// The consumer's error contract carries none of the producer's metrics. + /// The metric rule: the producer is a readout whose metric the + /// consumer does not accept on that input; or, on a readout's own input, + /// the readout's metric is not one its output accepts. MetricRule { consumer: CandidateForMemoGroupId, input: usize, producer_metric: AccuracyMetric, - consumer_carries: Vec, + /// The metrics accepted on that input (on a readout's own input: at + /// its output). + accepted_metrics: Vec, + }, + /// The consumer is a top-k readout whose `k` exceeds the producer's + /// heap size: the heap cannot hold the k ranked keys, whatever accuracy + /// is required. + HeapSmallerThanTopK { + consumer: CandidateForMemoGroupId, + input: usize, + k: usize, + }, + /// The producer is a readout with an accuracy metric but no guarantee + /// evaluable at the required failure probability (an HLL distinct + /// count, whose failure probability is unknown), or with no guarantee + /// at all (UnivMon's distinct count), read by a query with an accuracy + /// target: its error has no bound. + NoGuarantee { + consumer: CandidateForMemoGroupId, + input: usize, + }, + /// The readout `consumer` is reached by query `query`, whose target sets + /// a failure probability over the whole result while the number of + /// values one evaluation returns ($n_q$) is unknown: no per-value + /// failure probability follows, so no state meets it; and no other + /// query reaching it gives it a level. + ValuesPerEvaluationUnknown { + consumer: CandidateForMemoGroupId, + query: usize, + }, + /// The producer is a candidate over a tumbling-window chain whose width + /// does not divide the consumer group's evaluation period (or the group + /// has no known evaluation grid): the chain's windows do not end at the + /// consumer's evaluation times. + ChainWidthDoesNotDivideEvaluationGrid { + consumer: CandidateForMemoGroupId, + input: usize, + tumbling_window_width_ms: u64, + evaluation_period_ms: Option, }, /// The candidate never had a consumer and is not a query output. A /// candidate whose consumers were eliminated is traced to their causes. diff --git a/crates/types/src/summary_type/parameter_lists.rs b/crates/types/src/summary_type/parameter_lists.rs index 5ca4c3b62..3e877bbf5 100644 --- a/crates/types/src/summary_type/parameter_lists.rs +++ b/crates/types/src/summary_type/parameter_lists.rs @@ -7,10 +7,10 @@ use super::{SketchAlgorithm, SketchParams}; /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). /// -/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing -/// (`asap_accuracy`'s test-only `guarantee::sizing_oracle`) over the errors ε = 0.1, -/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without -/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// The widths, `k`s, precisions, `alpha`s and heap sizes span the errors +/// 0.1 down to 0.001, smallest structure first, without repeats; an +/// `asap_accuracy` test checks every entry has a guarantee at δ = 0.01. +/// Distinct counts (KMV, /// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by /// a structural parameter, the list covers the δ levels with it: Count-Min /// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with diff --git a/crates/types/src/summary_type/statistic.rs b/crates/types/src/summary_type/statistic.rs index 3dc78475f..bac12ec25 100644 --- a/crates/types/src/summary_type/statistic.rs +++ b/crates/types/src/summary_type/statistic.rs @@ -138,6 +138,25 @@ pub enum SketchStatistic { TopK { k: usize }, } +impl SketchStatistic { + /// The statistic a summary is read for to answer `intent`, or `None` + /// when no sketch statistic serves the intent. + pub fn for_intent(intent: &AggIntent) -> Option { + Some(match intent { + AggIntent::Quantile { q, .. } => Self::Quantile { q: *q }, + AggIntent::Cardinality { .. } => Self::Cardinality, + AggIntent::FrequencyL2 { .. } => Self::FrequencyL2, + AggIntent::FrequencyEntropy { .. } => Self::FrequencyEntropy, + AggIntent::TopK { k, .. } => Self::TopK { k: *k }, + AggIntent::Count => Self::PointCount { + key: ColumnRef::SampleValue, + value: None, + }, + _ => return None, + }) + } +} + /// The metric whose raw data samples, or plain sums of them, are every value /// `input` outputs, or `None`: walking down through a time range, an /// `offset`-only time shift (it moves the window, not the samples) and a diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index d55290ee6..c8670a170 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -4,6 +4,7 @@ //! to the crate-private function of the same name, except the fixture //! [`population_over_selector`], which the executor's tests build. +pub mod memo_builders; pub mod memo_fixtures; use crate::ir::operator::maintained_population::CurrentSeriesInput; diff --git a/crates/types/src/test_support/memo_builders.rs b/crates/types/src/test_support/memo_builders.rs new file mode 100644 index 000000000..c3088e786 --- /dev/null +++ b/crates/types/src/test_support/memo_builders.rs @@ -0,0 +1,213 @@ +//! Memo builders the memo's unit tests and the stage and accuracy crates' +//! tests share: each adds groups with the candidates Pass 1 would give +//! them (`Memo::find_or_add_group`, `Memo::add_candidate`) and queries +//! (`Memo::add_query`), so no test calls a stage crate. The expressions +//! come from [`memo_fixtures`](super::memo_fixtures). + +use crate::accuracy::AccuracyTarget; +use crate::ir::operator::arguments::Reduction; +use crate::ir::operator::AggIntent; +use crate::ir::schema::series_identity::with_promql_series_identity; +use crate::ir::schema::{DataType, FieldDataType}; +use crate::ir::{NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr}; +use crate::stages::{ + listed_build_parameters, CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, Memo, + MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, +}; +use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use crate::workload::{DataWorkload, SourceFacts}; + +use super::memo_fixtures::{range, readout, scan, state_grouped_by}; + +/// Adds `operator` with `parameters` to `group`, as Pass 1 does. +pub fn try_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> Result { + memo.add_candidate( + group, + CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, + Vec::new(), + ), + ) +} + +/// [`try_candidate`], which must succeed. +pub fn candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> CandidateForMemoGroupId { + try_candidate(memo, group, operator, parameters).unwrap() +} + +/// The group of `operator`, with `operator` as its exact candidate. +pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + candidate(memo, group, operator, None); + group +} + +/// The [`group`] of a `TimeSeries(metric)` scan listing Utf8 `labels` +/// whose label matchers are `matchers`, its series identity realized as +/// the planner realizes it before Stage 0 (`with_promql_series_identity`): +/// a closed schema whose last column is `$promql_series_identity`, keyed by +/// it and the timestamp. +pub fn scan_group( + memo: &mut Memo, + metric: &str, + labels: &[&str], + matchers: Vec>, +) -> MemoGroupId { + let labels: Vec<(&str, DataType)> = labels.iter().map(|l| (*l, DataType::Utf8)).collect(); + let mut operator = scan(metric, &labels, matchers); + let open = + OperatorNode::new_shared(operator.map_children(|_| unreachable!("a scan has no input"))) + .unwrap(); + let realized = with_promql_series_identity(&open).unwrap().schema.clone(); + if let Operator::NonASAP(NonASAPOp::Scan { schema, .. }) = &mut operator { + *schema = realized; + } + group(memo, operator) +} + +/// Adds the query rooted at group `root` with `target` (`None`: accuracy +/// not enforced), returning one value per evaluation. +pub fn query(memo: &mut Memo, root: MemoGroupId, target: Option) -> usize { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: target, + number_of_values_per_query_evaluation: Some(1), + }) +} + +/// Candidate 0 of `group`: the exact candidate [`group`] adds first. +pub fn own_candidate(group: MemoGroupId) -> CandidateForMemoGroupId { + CandidateForMemoGroupId { + group, + index_in_group: 0, + } +} + +/// A data workload declaring only that `metric` holds `series_count` +/// series. +pub fn with_series(metric: &str, series_count: u64) -> DataWorkload { + DataWorkload { + sources: [( + metric.to_string(), + SourceFacts { + series_count: Some(series_count), + ..SourceFacts::default() + }, + )] + .into(), + ..DataWorkload::default() + } +} + +/// The state group of a sketch build of `algorithm` over `child` grouped +/// by `reduction`, one build candidate per listed θ +/// (`listed_build_parameters`). +pub fn sketch_state( + memo: &mut Memo, + child: MemoGroupId, + algorithm: SketchAlgorithm, + reduction: Reduction, +) -> MemoGroupId { + let operator = state_grouped_by(FieldDataType::Sketch(algorithm), reduction, child); + let state = memo.find_or_add_group(operator.clone()).unwrap(); + for parameters in listed_build_parameters(&operator) { + candidate(memo, state, operator.clone(), parameters); + } + state +} + +/// The [`group`] of a readout of `statistic` over `state`. +pub fn estimate(memo: &mut Memo, state: MemoGroupId, statistic: SketchStatistic) -> MemoGroupId { + group(memo, readout(statistic, state)) +} + +/// `quantile_over_time(q, m[seconds])` per series of `m` by `job` through +/// a sketch state of `algorithm`: the range group, the state group and the +/// readout group. +pub fn quantile_over_time( + memo: &mut Memo, + algorithm: SketchAlgorithm, + q: f64, + seconds: u64, +) -> (MemoGroupId, MemoGroupId, MemoGroupId) { + let input = scan_group(memo, "m", &["job"], vec![]); + let window = group(memo, range(seconds as i64 * 1000, input)); + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + (window, state, readout) +} + +/// The readouts Pass 1 offers one quantile group, per summary type. +pub struct QuantileReadouts { + /// The `Aggregate` group `quantile_over_time(q, window)` per series. + pub result: MemoGroupId, + /// Per algorithm: its state group and the readout candidate of `result`. + pub by_algorithm: Vec<(SketchAlgorithm, MemoGroupId, CandidateForMemoGroupId)>, +} + +/// `quantile_over_time(q, window)` per series as Pass 1 leaves it: the +/// aggregate group with its exact candidate and, per algorithm, a state +/// group (one build per listed θ) and a candidate of the aggregate group +/// reading it (through a renaming `Project` over a readout group when the +/// column names differ, as Pass 1 does). +pub fn quantile_over_time_readouts( + memo: &mut Memo, + window: MemoGroupId, + q: f64, + algorithms: &[SketchAlgorithm], +) -> QuantileReadouts { + let result = group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: window, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Quantile { col: None, q }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ); + let mut by_algorithm = Vec::new(); + for &algorithm in algorithms { + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let direct = readout(SketchStatistic::Quantile { q }, state); + let id = match try_candidate(memo, result, direct, None) { + Ok(id) => id, + Err(CandidateError::OutputDiffersFromGroup { .. }) => { + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + let to = &memo.groups()[result.0 as usize].group_properties.schema; + let renamed = Operator::NonASAP(NonASAPOp::Project { + cols: to + .fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + .collect(), + child: readout, + }); + candidate(memo, result, renamed, None) + } + Err(error) => panic!("{error}"), + }; + by_algorithm.push((algorithm, state, id)); + } + QuantileReadouts { + result, + by_algorithm, + } +} diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs index 4e494efa0..9a23b9a4e 100644 --- a/crates/types/src/test_support/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -1,13 +1,21 @@ -//! Memo expressions over group ids that the memo's unit tests and this +//! Memo expressions over group ids that the memo's unit tests, this //! crate's memo integration tests (`memo_group_properties`, -//! `memo_aggregate_coverage`) build their examples from. +//! `memo_aggregate_coverage`) and the stage crates' tests build their +//! examples from. The builders that add them to a memo are in +//! [`memo_builders`](super::memo_builders). use std::time::Duration; -use crate::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn, TimeShift}; +use crate::ir::operator::arguments::{ + AtModifier, BinaryOpKind, Reduction, Source, SourceColumn, TimeShift, +}; use crate::ir::operator::AggIntent; -use crate::ir::scalar::ColumnRef; +use crate::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind}; +use crate::ir::scalar::{ColumnRef, ScalarValue}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; -use crate::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use crate::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, + TimeRangeKind, +}; use crate::stages::MemoGroupId; use crate::summary_type::{SketchStatistic, SummaryUpdate}; @@ -85,11 +93,20 @@ pub fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Oper /// `SummaryAgg[summary_type]` of the sample value per series. pub fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + state_grouped_by(summary_type, Reduction::PerEntity, child) +} + +/// `SummaryAgg[summary_type]` of the sample value, grouped by `reduction`. +pub fn state_grouped_by( + summary_type: FieldDataType, + reduction: Reduction, + child: MemoGroupId, +) -> Operator { Operator::ASAP(ASAPOp::SummaryAgg { child, summary_type, input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, + reduction, filter: None, }) } @@ -133,3 +150,38 @@ pub fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator literal`, with `semantics` (the frontend's comparison +/// rules). +pub fn compare( + column: usize, + op: CompareOpKind, + literal: ScalarValue, + semantics: ExprSemantics, +) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(literal)), + semantics, + } +} + +/// `lhs rhs`, value by value, without vector matching. +pub fn arithmetic( + kind: ArithmeticOpKind, + lhs: MemoGroupId, + rhs: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(kind), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) +} diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index f7ea803bd..42d16a762 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -142,10 +142,12 @@ pub struct SourceFacts { /// it must hold for the longest evaluation window any of them uses; a /// filter or a shorter window only lowers it. Each group's own items /// count separately: the bound is per output row. The deployment must - /// enforce it; an observed distinct count is not a bound. Plan - /// selection binds it to give classic HLL (and a distinct-count UnivMon) - /// its failure probability per output row; without it that probability - /// is unknown. KMV's guarantee needs no bound. + /// enforce it; an observed distinct count is not a bound. It is for a + /// distinct-count guarantee whose failure probability per output row + /// depends on the number of distinct items (classic HLL, a + /// distinct-count UnivMon); no guarantee in `asap_accuracy` reads it + /// yet, so those readouts serve no query with an accuracy target. KMV's + /// guarantee needs no bound. #[serde(default)] pub max_distinct_per_evaluation: Option, /// How many distinct values each label (PromQL) or column (SQL) of the diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs index 94bc495db..7f1f4caba 100644 --- a/crates/types/tests/column_lineage.rs +++ b/crates/types/tests/column_lineage.rs @@ -19,8 +19,8 @@ use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, }; use asap_types::stages::{ - CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, - LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, + CandidateForMemoGroup, ColumnLineage, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + ValueSign, }; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{ @@ -150,14 +150,8 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { ], child: g5, }); - let expression = CandidateExpression { operator }; - let candidate = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; + let candidate = + CandidateForMemoGroup::new(operator, None, OriginOfCandidateForMemoGroup::Pass1, vec![]); let id = memo.add_candidate(g3, candidate).unwrap(); assert_eq!(id.group, g3); } diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 5c96ed40b..5a2b74d2d 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -18,12 +18,14 @@ use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarVal use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; +use asap_types::plan::view_coverage; use asap_types::stages::{ - view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, ColumnLineage, - CoverageError, LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, - OriginOfCandidateForMemoGroup, + AggregateCoverage, CandidateForMemoGroup, ColumnLineage, CoverageError, Memo, MemoGroupError, + MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{ + ExactKind, MergeInputRule, SketchAlgorithm, SketchParams, SummaryUpdate, }; -use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{filter, range, scan, series_column, shift, state}; use asap_types::workload::DataWorkload; @@ -350,7 +352,7 @@ fn scan_predicates_lift_and_their_states_merge() { /// G1 Scan latency /// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) /// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) -/// G4 candidate: SummaryMerge(G7) +/// G4 candidate: SummaryMerge[k=269](G7) /// ``` #[test] fn a_merge_over_a_chains_windows_covers_the_direct_build() { @@ -370,16 +372,12 @@ fn a_merge_over_a_chains_windows_covers_the_direct_build() { assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); - let expression = CandidateExpression { - operator: merge(vec![g7]), - }; - let merge_of_windows = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), - expression, - input_requirements: vec![LogicalRequirements::default()], - fused_groups: vec![], - origin: OriginOfCandidateForMemoGroup::Pass1, - }; + let merge_of_windows = CandidateForMemoGroup::new( + merge(vec![g7]), + Some(SketchParams::Kll { k: 269 }), + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); memo.add_candidate(g4, merge_of_windows).unwrap(); let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 37acefdae..d4ba8d6fa 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -294,7 +294,6 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { lifetime: Lifetime::None, parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, @@ -310,7 +309,6 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { lifetime: Lifetime::None, parameters: None, output_schema: schema.clone(), - guarantee: None, retention_ms: None, selected_physical_candidate: None, view_coverage: None, diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 95e171a0a..bb98476ab 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,4 +1,3 @@ -use asap_types::accuracy::AccuracyGuarantee; use asap_types::ir::operator::{AggIntent, Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -57,15 +56,14 @@ fn rebuilding_rederives_schema_for_both_categories() { } } -/// Naming overrides survive rewrites without freezing types or assessed properties. +/// Naming overrides survive rewrites without freezing types. #[test] fn rebuilding_preserves_only_explicit_naming_overrides() { for asap in [false, true] { let original = aggregate(scan(DataType::Int64, "key"), asap); let mut schema = original.schema.clone(); schema.fields[0].name = "alias".into(); - let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); + let renamed = OperatorNode::with_schema(original.operator.clone(), schema); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -75,7 +73,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { rebuilt.schema.fields[0].plain_dtype(), Some(&DataType::Utf8) ); - assert!(rebuilt.guarantee.is_none()); rebuilt.validate_structure().unwrap(); } } diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 16e76b000..92a9c0adb 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -152,7 +152,6 @@ fn shared_construction_derives_both_operator_categories() { ); assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); - assert!(state.guarantee.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index b1f7c8b2e..1b42dac39 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -9,7 +9,7 @@ of its inputs. A locally accurate sketch does not automatically make an entire query accurate. Planner owns the interpretation of accuracy targets, estimator models, error -propagation, budget allocation and candidate legality. Deployments provide +propagation, readout requirements and candidate legality. Deployments provide source contracts and evidence with a defensible scope. Cost models compare resource costs; runtime implementations execute the selected semantics and must honor the conditions attached to them. @@ -26,7 +26,7 @@ These three concepts have different roles: | Concept | Meaning | Owner | |---|---|---| | Accuracy target | What the caller requires of the result | Query requirements and aggregate intent | -| Result guarantee | What a particular computation can establish | Estimator and composition models in Planner | +| Result guarantee | What a particular computation can establish | Estimator models and error propagation in Planner | | Evidence | Facts or trusted contracts needed to establish that guarantee | Source or deployment evidence provider | `AccuracyTarget` is the authoritative requirement type: @@ -65,25 +65,32 @@ share meets a target over a larger share but not a per-value target, and only a per-value guarantee is restated over a scope. A scope bounds only the failure probability; the error bound is unchanged. -Every estimator and composition produces a per-value guarantee (for a grouped +Every estimator produces a per-value guarantee (for a grouped query, one per group: a Kmv or HLL distinct count per group, a KLL or DDSketch quantile per group, a Count-Min or Count Sketch frequency per key, a Hydra estimate per subpopulation; an exact value fails with probability 0). Stated over a scope, a per-value failure probability `δ` becomes `V · δ` by the union bound, without assuming independence, so a sketch shared by several groups (Hydra, a keyed Count-Min) is covered the same way -(`AccuracyGuarantee::over_scope`). A guarantee whose own failure probability is -unknown (generic HLL) stays unmet under every scope with a δ, and top-k -membership needs its own evidence under every scope. - -Pass 1 sizes each approximate input of a candidate with `n` accuracy-budget -parts for `δ / (n · V)`, so the union bound gives δ over the covered values. -A candidate several queries reach is sized for the smallest `δ / (n · V)` -among them, and a build Pass 2 shares among several readers takes the -strictest reader's size. Stage 3 only checks the composed guarantee; it never -resizes. At equal δ and `n`, the scope covering the most values is the -strictest: every group (`G`, when `G` is above 100), then a 1% share (100), -then a 5% share (20), then a single value (1). A whole-result target with an unbound `G` +(`AccuracyScope::max_values_covered` gives `V`). A guarantee whose own failure probability is +unknown (generic HLL) serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`), and a top-k readout is admitted on its +heap size and its value guarantee alone (the contracts table). + +Pass 2 gives each sketch readout the error and per-value failure probability +`δ / max(V, R)` it must meet, with `R` the readout group's values per +evaluation (the series its scans read): a `max` over 1000 readouts returns +one value yet fails when any readout does, so the union bound gives δ over +every readout read; +Pass 1's candidate takes the first requirement level and Pass 2 adds a copy +per further level. A readout several queries reach gets the union of their +levels. Pass 2 sizes nothing: a state has one build per listed θ, and each +edge checks the guarantee of its producer's θ at the required failure +probability, so a build several readers share serves the readers whose +levels its θ meets. Stage 3 only checks the guarantee; it never resizes. At equal +δ, the scope covering the most values is the strictest: every group (`G`, +when `G` is above 100), then a 1% share (100), then a 5% share (20), then a +single value (1). A whole-result target with an unbound `G` enlarges nothing and stays unmet. A grouped result's `G` is bound by the distinct values its sources declare for its group keys (`SourceFacts.distinct_values_per_label`, and @@ -94,7 +101,7 @@ most the series read. With each of 625 endpoints in one of 25 services and most 625 values, not 25 x 625. A key counts only when it is a source column passed through unchanged; a relabeled or computed key leaves `G` unbound. So a sketch meets a whole-result target over SQL groups when every grouping column's distinct values are declared, and over 25 services -and 25 endpoints sizes each group's sketch for `δ / 625`. Nothing declared +and 25 endpoints requires each group's sketch to meet `δ / 625`. Nothing declared leaves `G` unbound; no default stands in for it. An `AccuracyGuarantee` contains an error metric, an error-bound expression, a @@ -116,8 +123,8 @@ rule and any evidence that rule needs. Bounds and probabilities can remain symbolic. An unknown population size, normalization factor or failure probability stays unknown; it is not replaced -by zero. Provenance records estimator parameters and contracts, child -guarantees, composition steps, allocated targets and unavailable evidence. +by zero. Provenance records that a value is exact, or the sketch evaluation with its +estimator parameters and contract. This makes an accuracy decision explainable without reconstructing it from query text or cost estimates. @@ -125,16 +132,16 @@ query text or cost estimates. ```mermaid flowchart TD - Request[Query semantics and accuracy target] --> Generate[Generate candidates and size parameters] + Request[Query semantics and accuracy target] --> Generate[Generate candidates with listed parameters] Evidence[Scoped source contracts and evidence] --> Generate Generate --> Local[Derive local evaluation guarantees] Evidence --> Local - Local --> Compose[Propagate guarantees through the DAG] + Local --> Compose[Propagate required errors through the DAG] Evidence --> Compose Compose --> Check[Check the composed target] Check -->|Satisfied with sufficient evidence| Eligible[Eligible for cost comparison] Check -->|Evidence missing| Pending[Retain an uncertified candidate for explanation] - Check -->|Known violation or unsupported composition| Reject[Reject with a reason] + Check -->|Known violation or unsupported propagation| Reject[Reject with a reason] Eligible --> Select[Select compatible candidates across the workload] Select --> Bind[Bind implementations and deployment resources] ``` @@ -156,26 +163,26 @@ missing-evidence gate still matters during selection. A successful accuracy check also does not prove that a runtime implements the plan, that stored state is ready, or that a complete deployment cost is available. -## Local estimator models and parameter sizing +## Local estimator models and parameter lists A local model describes a specific evaluation of a specific estimator with committed parameters and applicable assumptions. A family name or a parameter such as HLL precision is not, by itself, a confidence certificate. -Sizing proposes parameters for a local target. Planner then derives the -actual guarantee from those parameters and checks it. Parameter rounding, -implementation limits, or conservative probability bounds can make a proposed -configuration insufficient; sizing must not bypass that check. +Planner does not size parameters for a target. Each algorithm has a listed +set of parameters θ; Planner derives the guarantee of each listed θ at the +required failure probability and checks it against the requirement. +Implementation limits or conservative probability bounds can make a listed θ +insufficient for a requirement; that θ then serves no reader that needs more. The built-in models currently include: | Model | Current accuracy contract and limits | |---|---| -| Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require propagation | +| Exact computation | Exact over exact inputs under the supported operation's semantics; approximate inputs still require error propagation | | KLL | Normalized rank error from the empirical 99% calibration `2.296 / k^0.9723`, fitted on Apache DataSketches' C++ KLL (`a9b42755072b`); below a failure probability of 0.01 that error is scaled by the Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`, a model rather than a measurement, and the guarantee names that contract apart | -| DDSketch | Relative value error from alpha under the supported estimator/domain contract; sensitive compositions require additional domain evidence | +| DDSketch | Relative value error from alpha under the supported estimator/domain contract; operators sensitive to sign or range require additional domain evidence | | Generic HLL | RSE magnitude with unknown failure probability; not a general confidence theorem | -| Bounded Classic HLL | Source-conditioned relative-error/failure-probability bound and precision sizing for the linear-counting branch | | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV | Provable distinct-count bound from the `k` smallest hashes | @@ -188,82 +195,105 @@ another implementation with the same algorithm name. In particular, a named empirical calibration is different from an arbitrary benchmark's maximum observed error; both its confidence and applicability must remain explicit. -Sizing is each algorithm's predefined parameter list, `parameter_list` -(`crates/types/src/summary_type/parameter_lists.rs`), which a test keeps equal -to the estimator modules' closed-form sizing over an error grid; a summary shared by several readers is sized with each -parameter's largest value among them. Where an error bound grows as the failure -probability shrinks (a KLL's `k`, a distinct-count UnivMon's heap), each -reader's parameter is first resized to the readers' smallest failure -probability at no larger error, so the shared summary meets every reader at -the one failure probability it states. Accuracy validation is independent of -sizing: Stage 3 checks the guarantee derived from the committed parameters. -The design boundary is that sizing never grants accuracy authority to the -cost model. +The θ lists are each algorithm's predefined parameter list, `parameter_list` +(`crates/types/src/summary_type/parameter_lists.rs`), weakest first. Pass 1 +and Pass 2 add one build per listed θ, and the logical-edge check keeps an +edge only when the guarantee of the producer's θ, at the readout's required +failure probability, is within the required error. Where an error bound grows +as the failure probability shrinks (a KLL's `k`), the same θ meets a reader +at 1e-2 and misses one at 1e-6; no θ is resized for a reader. Accuracy +validation is independent of cost: the guarantee is derived from the +committed parameters, and the cost model never grants accuracy authority. -## Composing guarantees through a DAG +## Propagating errors through a DAG The accuracy crate supplies theoretical guarantees (`theoretical_guarantee`), -propagation rules and the comparison limit of a target (`target_limit`); the -deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a -theoretical guarantee. The rules are conservative: an unregistered -composition over approximate inputs is rejected rather than treated as exact. - -The principal composition rules are: - -| Computation | Accuracy reasoning | +each operator's error contract (`error_propagation`) and the comparison limit +of a target (`target_limit`); the deployment's measurements +(`AccuracyEvidence::measured_guarantee`) take precedence over a theoretical +guarantee. Only a sketch readout introduces error. Every other operator +passes its inputs' errors on through a contract, and the contracts are +conservative: an operator with no contract for a metric fails the edge of an +input whose error is in that metric (`RootPruneReason::MetricRule`), so an +approximate value is never treated as exact. + +A contract is a linear bound: the consumer's output error is at most +`Σ β_i · ε_i` over its inputs' errors `ε_i`, valid while each input stays +within its validity cap `b_i`. The principal contracts are: + +| Operator | Contract | |---|---| -| Supported computation over exact inputs | Only its own local estimator error remains; a supported exact operation remains exact | -| Additive absolute-error composition | Add compatible error bounds and union-bound their failure events | -| Relative-error composition | Include the multiplicative cross term, subject to the required sign/domain conditions | -| Registered Lipschitz transform | Scale the input bound by the registered constant and include local error | -| Exact sum over approximate values | Convert compatible errors to absolute units and account for input multiplicity; missing scale/count evidence remains unknown | -| Exact average or extremum over approximate values | Use the registered absolute-error bound and account for failures across the input population | -| Division | Two relative errors (or an exact numerator over a relative error) compose in closed form, `(a + b)/(1 − b)` for `b < 1`, with no domain evidence; any other approximate denominator needs enforced operand domains bounded away from zero; rank-error division is unsupported | -| Counter rate/increase over approximate samples | No general distribution-free rule for reset detection and extrapolation; exact inputs remain a distinct supported case | -| Candidate-based TopK | Require membership/completeness evidence; frequency accuracy and exact reranking alone are insufficient | - -Probability composition uses union bounds, without assuming independence -between child errors. Estimator-local assumptions, such as independent bucket -hashing, must be stated separately. Shared DAG nodes do not create independent -errors merely because several consumers reference them, and sharing does not -justify reducing the failure budget. Per-result guarantees also do not imply -a simultaneous guarantee across every query or evaluation time in a dashboard. - -For relative errors, two eligible layers with errors `e1` and `e2` compose as -`(1 + e1)(1 + e2) - 1`, not simply `e1 + e2`. A ratio of relative errors `a` -over `b` is within `max((1 + a)/(1 − b) − 1, 1 − (1 − a)/(1 + b)) = (a + b)/(1 − b)`. -A relative bound below 1 keeps zero at zero and keeps the sign, so a zero -denominator gives the same non-finite answer exactly and approximately. The -rule relies on each operand's own relative guarantee (for DDSketch, its -estimator contract: one-signed values within the mapping's range). A ratio -of operands whose relative bound is still symbolic in Stage 1 (e.g. a sum of -quantiles gated on the values' sign) gets no guarantee. A sum or difference of relative -errors has no relative bound (cancellation), and a rank error (KLL) has no -value bound, so neither composes through division. Likewise, an exact sum above -approximate values retains their uncertainty: exact arithmetic does not -recover information lost below it. - -## Allocating an end-to-end budget - -`AccuracyBudgetAllocator` proposes local targets for a composition. These -allocations create candidates; they are not proofs that the candidates work. -Every proposed composition is checked again using the actual guarantees. - -The default equal-split allocator divides an additive error budget across the -approximate layers. For relative error, it uses local error -`1 - (1 + epsilon)^(-1/n)` for `n` layers. The true error of any mix of -products and quotients stays within the target (each layer contributes at most -`1/(1 - e)`); the composed bound meets it for one product or quotient of two -layers, not for a nested quotient or an exact numerator over a lone -approximate denominator. When the target includes delta, it -splits that budget across the layers for union-bound composition. An exact -target does not receive an approximate allocation. - -For example, under a supported two-layer relative-error composition, a 10% -end-to-end budget gives each layer approximately 4.65%, not 5%. If the failure -budget is 1%, each layer receives 0.5%. A model with only a fixed 1% failure -contract cannot automatically satisfy that allocation; another supported -configuration or candidate is needed. +| Sort, TimeRange, TimeShift, Concat, a readout, a finalize, a summary merge | `β = 1` in any metric | +| Filter on columns copied from a source (labels and keys) | `β = 1` in any metric; a filter comparing a computed value has none | +| Project | Judged on the items reading a computed value (labels, the time and the series identity are copied): `β = 1` in any metric for a copied column; absolute error, `β = 1`, for `abs`, `clamp`, `clamp_min`, `clamp_max` (also PromQL's `promql_` functions) and a negation; relative error, `β = 1`, for a product with or quotient by a nonzero constant | +| Aggregate or exact-accumulator build of counts, min, max, avg | Counts `β = 0`; min and max absolute or relative with `β = 1`; avg absolute with `β = 1`; only when every group-by key is copied from a source (a `by` key as listed; for `without`, every column but the time, the sample value and the excluded ones); none with measure filters or `HAVING`, or with a key reading an estimate | +| `+`, `-` | Absolute error, `β = 1` | +| `/` | Relative error, `β = 1/(1 − b)` on both operands, the denominator capped at `b` | +| `*` | Relative error, `β = (1 + b, 1)`, the right operand capped at `b` | +| Sum, comparison, ranking, counter rate or increase, any other function | No contract: the edge of an approximate input fails | +| Top-k over a sketch | Checked: the heap size is at least `k`, and the readout's value guarantee meets the requirement; membership has no check of its own | + +`b` is `RELATIVE_OPERAND_VALIDITY_CAP`, 0.1, the weakest error of every +algorithm's list, so it never excludes a listed θ. For relative errors `x` +and `y` with `y ≤ b`, `(1 + x)(1 + y) − 1 ≤ (1 + b)·x + y`, and a ratio of +relative errors `a` over `b` is within `(a + b)/(1 − b)`. A relative bound +below 1 keeps zero at zero and keeps the sign, so a zero denominator gives the +same non-finite answer exactly and approximately. The ratio rule relies on +each operand's own relative guarantee (for DDSketch, its estimator contract: +one-signed values within the mapping's range). A sum or difference of +relative errors has no relative bound (cancellation), and a rank error (KLL) +has no value bound, so neither passes through division. Likewise, an exact +sum above approximate values retains their uncertainty: exact arithmetic does +not recover information lost below it. + +Probability is not propagated through operators. A query's failure +probability is split into a per-value one for each readout; a composed query +whose readouts' failure events add up (a ratio of two quantiles) reads each at +a level of a failure-probability grid, and the levels sum within the query's +per-value failure probability by the union bound, without assuming +independence between child errors. Estimator-local assumptions, such as +independent bucket hashing, must be stated separately. Shared DAG nodes do +not create independent errors merely because several consumers reference +them, and sharing does not justify reducing the failure budget. Per-result +guarantees also do not imply a simultaneous guarantee across every query or +evaluation time in a dashboard. + +## Readout requirements for composed queries + +Pass 2 gives each readout the accuracy it is required to meet. A query's +target error is divided by the factors `β` of the consumers on the path from +the readout to the query root, and capped by the validity caps on the way: the +largest error the readout's output may have for that query. + +- A query whose path reads one readout gives one requirement level: the + target divided by the path's factor, at the target's failure probability per + value. +- A composed query (several readouts, such as a ratio) gives every level of + the error grid (10%, 5%, 2%, 1%, 0.5%, 0.2%, 0.1%) and the failure-probability + grid (powers of ten from 1e-2 down to 1e-5, continued down to the first + one at or below a tenth of the query's per-value failure probability) not + looser than the cap. The logical-edge check then + keeps the levels a producer's θ meets. +- A readout reached by several queries gets the union of their levels. +- A count reads no value (`β = 0`): the readouts below it get no level + through it. +- A query whose target sets a failure probability with no per-value one + (`n_q` or the readout's values per evaluation unknown) gives no level. A + readout no query gives a level is pruned + (`RootPruneReason::ValuesPerEvaluationUnknown`); one another query gives a + level stays, and the query reads its group only exactly. + +The metric rule cuts a readout's edge when no query reaching it accepts its +metric, a union over the queries. Per query, Pass 2 also lists the groups it +reaches through a consumer without a contract for its target's metric +(`LogicalASAPMemoCandidates::exact_only_groups_of_query`): with Q0 = +`p99 / p50` and Q1 = `p99 / p50 > 1` sharing the ratio, Q0 may read the +DDSketch readouts while Q1 reads the ratio and every group below it only +exactly. Stage 3 must enforce the list per query; Pass 2 only computes it. + +An exact target gets no approximate level. A model with only a fixed 1% +failure contract cannot meet a level that needs a smaller failure probability; +another supported configuration or candidate is needed. ## Evidence and trust boundaries @@ -285,30 +315,16 @@ certification. If a model uses an empirical accuracy calibration, the contract must identify its confidence level and scope rather than silently promoting an observation into a guarantee. -### Example: bounded Classic HLL - -A deployment declares `DataWorkload.sources[source].max_distinct_per_evaluation` -for the source the distinct count reads. It asserts an enforced maximum -distinct population per evaluation, including all merged tumbling windows; the classic -estimator and independent uniform bucket hashing are the model's -assumptions. Stage 1 sizes the precision from the query or allocated local -target and leaves the failure probability symbolic, as -`classic_hll_failure_probability[p=P,eps=ε](max_distinct[source])`. Pass 1 -then raises the precision when the source's declared bound needs it (for its -share of the failure probability), and Stage 3 binds the failure probability -from that bound in the normal selection checks. A bound outside the -contract's range gives a failure probability of 1; a missing bound leaves it -unknown, so the target is not met. - -The current model supports maxima from 1 to 131,072 (2^17) and precisions from 4 to 18, -and certifies only configurations that remain in the linear-counting branch. -It bounds collisions across every integer cardinality in the declared domain -and bounds overestimation deterministically. It is not an RSE-to-normal -conversion, nor does it cover HIP/MLE or arbitrary unbounded populations. - -Missing evidence leaves generic HLL confidence unknown. Invalid or infeasible -contracts cannot authorize the result. The contract does not certify another -estimator, another expression or an unsupported shared-grid grouping. +### Example: HLL distinct counts + +HLL's guarantee is its relative standard error, `1.04 / √m` for `m` buckets, +with an unknown failure probability. Its readout's metric is relative error, +but no failure probability can be evaluated, so a distinct count read from +HLL serves no query with an accuracy target +(`RootPruneReason::NoGuarantee`); KMV's distinct count, whose bound does not +depend on the number of distinct items, serves such a query. Missing evidence +leaves generic HLL confidence unknown. Invalid or infeasible contracts cannot +authorize the result. ## Sharing across consumers @@ -343,12 +359,12 @@ These percentages bound rank displacement, not latency-value error. ```mermaid flowchart LR - Input[Same source, filter and window] --> Shared[KLL computation per service sized for A] + Input[Same source, filter and window] --> Shared[KLL computation per service meeting A] Shared --> A[Consumer A: 1% rank error] Shared --> B[Consumer B: 5% rank error] ``` -B's independently sized candidate remains available. Global selection compares +B's own candidate, a θ meeting only B, remains available. Global selection compares that candidate with the reuse candidate and preserves the dependency on A's tighter computation if it chooses reuse. Sharing is therefore an explicit planning choice, not an unconditional rewrite or two separately charged builds. @@ -359,18 +375,10 @@ not just accuracy. Nor does this example justify serving B at a failure target of 0.1%; A's 1% failure guarantee would not meet it. A separate supported reuse rule or a different estimator configuration would be needed in those cases. -When the consumers' failure targets differ, the summary-capability rule offers -such a configuration. A shared summary states one accuracy bound at one failure -probability, so it is sized at the smallest one. Each consumer's parameters are -resized to it at no larger error, and the largest is kept. Suppose A asks -for 1% rank error at a 1% failure probability (`k = 269` alone) and B for -5% at 0.01% (`k = 79` alone). At 0.01% the Gaussian scale is 1.51, so A's -`k = 269` would give 1.5% rank error there, which misses A's target. A's -`k` is therefore grown to 412 (rank error 0.997% at 0.01%), and the shared KLL is -`k = 412` at 0.01%, which meets both. A distinct-count UnivMon's heap is shared -the same way. Checking each consumer at its own failure probability against -a smaller shared summary would need a guarantee per consumer, which the -current composition does not carry. +When the consumers' failure targets differ, each reads the shared KLL at +its own required failure probability: a θ whose guarantee meets both +requirements serves both, and one meeting only the looser does not serve +the tighter consumer. No θ is resized for sharing. ## Organization and extension contract @@ -380,36 +388,44 @@ The `asap_accuracy` crate separates these responsibilities: accuracy/src/ ├── lib.rs # Re-exports: the public interfaces ├── evidence.rs # AccuracyEvidence: the deployment's measured accuracy -├── requirements.rs # producer_satisfies_logical_requirements (α^log), target_limit -├── error_propagation.rs # ErrorPropagation: a consumer's output error from its inputs' -├── propagation_stats.rs # Evidence and source contracts -├── composition.rs # Error propagation across computations -├── budget.rs # Splitting a query's accuracy budget across approximate inputs -├── function_rules.rs # Error composition of scalar functions over approximate inputs +├── error_propagation.rs # error_propagation: each operator's error contract per +│ # metric; metrics_carried +├── metric_rule.rs # accepted_metrics, metrics_accepted_on_inputs (the +│ # metric rule) +├── readout.rs # readout_metric, sketch_readout_metric, +│ # readout_metric_of_candidate: the metric of a +│ # readout's error +├── readout_caps.rs # readout_levels: caps through the error contracts, each +│ # readout's required-accuracy levels; MAX_ERROR_LEVELS +├── logical_edge.rs # producer_satisfies_logical_requirements (α^log); +│ # target_limit, the requirement comparison └── guarantee/ - ├── mod.rs # Family/evaluation dispatch and source-contract integration + ├── mod.rs # theoretical_guarantee (one lookup at a failure + │ # probability per algorithm) + ├── binomial.rs # binomial_upper_tail ├── kll.rs ├── ddsketch.rs - ├── hll.rs # Generic HLL and bounded Classic HLL + ├── hll.rs # Generic HLL (unknown failure probability) ├── cms.rs ├── count_sketch.rs ├── cardinality.rs # KMV model └── univmon.rs ``` -Each estimator module owns its local accuracy formulas and associated sizing -formulas. The unified entry point delegates to these models; it does not +Each estimator module owns its local accuracy formulas: the guarantee of a +listed θ at a failure probability. Unit tests sit next to the file they +test. The unified entry point delegates to these models; it does not contain a second copy of their mathematics. Serializable guarantee and metric types live in `asap-types` so planning, explanations and downstream binding share the same contract. -Adding an estimator or composition requires: +Adding an estimator or an operator's error contract requires: 1. A precisely defined error metric, estimator/evaluation semantics and assumptions. -2. Sizing behavior and a guarantee derived from the committed parameters, including unsupported parameter domains. +2. A θ list and a guarantee derived from each listed θ at a required failure probability, including unsupported parameter domains. 3. Explicit evidence requirements, population scope and provenance. 4. Propagation rules where supported; rejection or retained unknowns elsewhere. -5. Tests from target and evidence through sizing, composition and selection, including missing evidence and infeasible targets. +5. Tests from target and evidence through the guarantee at the required failure probability, error propagation and selection, including missing evidence and infeasible targets. Algorithm-specific mathematics belongs behind this common contract. It does not require a new selection rule for every sketch. A deployment's measured diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 6b7fe81d9..1eb2a3a54 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -112,7 +112,7 @@ Operator nodes carry no physical properties: the logical DAG has no timing or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) decides, for each sub-DAG, whether its output is materialized, and if so with which -[`PhysicalProperties`](../../../crates/types/src/stages/physical/properties.rs): a +[`PhysicalProperties`](../../../crates/types/src/stages/physical/properties/mod.rs): a timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index 995eb99a6..1883b0fac 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -75,7 +75,7 @@ Both current `Concat`-constructing call sites, and every consumer of test fixtures) — none of them builds a fresh `Concat` with a `Dedup` on top that this feature could remove. - **Every consumer of `Schema::unique_keys`** in the DAG, to check for a - cost beyond "a literal `Dedup` node": `share_common_sub_dags` (now `Memo::find_or_add_group`, `crates/types/src/stages/logical/memo/hash_consing.rs`) + cost beyond "a literal `Dedup` node": `share_common_sub_dags` (now `Memo::find_or_add_group`, `crates/types/src/stages/logical/memo/hash_consing/mod.rs`) (gates CSE producer-sharing on `Schema::has_unique_key()`) and `asap_aware_mapping::rollup::is_legal_rollup_source` (gates rollup-source legality the same way, on an *`Aggregate`'s* own output schema). Neither diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 84d7dd22e..eac375728 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -117,7 +117,7 @@ A node in the physical data will represent the data or summary instance, so a no | How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | | How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `GroupProperties::aggregate_coverage`, derived in `stages/logical/aggregate_coverage.rs` | §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, derived in `stages/logical/group/aggregate_coverage.rs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,7 +237,7 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -Coverage is a memo-group property, derived like an output schema and never declared; its types and derivation are in `crates/types/src/stages/logical/aggregate_coverage.rs`. The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`): `aggregate_coverage` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s, and the result, its inputs named by `MemoGroupId`, is stored in the group's `GroupProperties::aggregate_coverage`. Node trees have none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). +Coverage is a memo-group property, derived like an output schema and never declared; its types and derivation are in `crates/types/src/stages/logical/group/aggregate_coverage.rs`. The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`): `aggregate_coverage` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s, and the result, its inputs named by `MemoGroupId`, is stored in the group's `GroupProperties::aggregate_coverage`. Node trees have none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). ```rust /// What a summary state computes, with every lifted selection removed, @@ -951,7 +951,6 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) │ (θ, `SketchParams`, is on the candidate) §6.4 -└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 └── definition: AggregateDefinition { input, unlifted_operators } @@ -966,7 +965,7 @@ MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | | §6.6 Aggregate coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of aggregate memo groups | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. The memo derives the `SummaryAgg` group's `AggregateCoverage` (§6.6) and stores it in the group's `GroupProperties::aggregate_coverage`. +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5). The memo derives the `SummaryAgg` group's `AggregateCoverage` (§6.6) and stores it in the group's `GroupProperties::aggregate_coverage`. ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) @@ -982,9 +981,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, /// The output columns. Derived from `operator` and its children. pub schema: Schema, - /// The accuracy statement of the output, once known. `None` does not - /// mean exact. - pub guarantee: Option, } ``` @@ -1353,9 +1349,9 @@ pub struct CurrentSeriesInput { } ``` -A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. +A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It describes evaluation and finalized values, and `MaintainPopulation` values (whose population is exact); never sketch or accumulator state. No node type stores it today. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/group/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/group/column_properties.rs`; the plan's content id in `crates/types/src/plan/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index eedea2fb3..7af7f3696 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -283,7 +283,7 @@ one. It makes no implementation choice, so this section counts it as Stage 0 lists `memo_of_queries` with Pass 1. `memo_of_queries` (`crates/stage0-memo/src/lib.rs`) visits the frontends' roots children first and finds or adds each node's group with `Memo::find_or_add_group`, whose -expression equality is in `crates/types/src/stages/logical/memo/hash_consing.rs`. Two structurally equal nodes +expression equality is in `crates/types/src/stages/logical/memo/hash_consing/mod.rs`. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader of the shared node reads the rows it would read alone. Equal nodes are interned children first, so they always read one input, and a node is legal @@ -311,19 +311,9 @@ rule's origin, and gives each reader a candidate that reads that group. | `OriginOfCandidateForMemoGroup` | Added by | Candidates | |---|---|---| | `Pass1` | `pass1_memo` (`stage1-pass1-per-computation-candidates`: `computation_targets.rs`, `candidates/`) | Every Pass 1 way to compute the group's measures: exact, exact accumulator, each summary algorithm with one build per configuration of its predefined list (`parameter_list`), and fused computations. A fused candidate lists the groups of the operators it replaces in `fused_groups`; those groups keep their own candidates for their other readers. | -| `TumblingWindows` | Window composition: `add_local_window_composition` and `extend_local_window_composition` (`pass2/memo/window_candidates.rs`) | A window summary computed from tumbling windows, at each tumbling-window width the reader's evaluation grid allows. The tumbling-window chain stays private to the candidate. | -| `WidenedScans` | Scan widening: `add_scan_label_candidates` (`pass2/memo/rules/scan_labels.rs`) | A reader of a metric's scans widened to a shared label set, read through a `Project`. | -| `SummaryInput` | `factor_summary_inputs` (`pass2/memo/rules/summary_inputs.rs`) | Canonical value preprocessing feeding a summary build. | -| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`pass2/memo/rules/topk.rs`) | One ranking (`Limit(k_max)` over `Sort`), and each top-k reading its first `k` rows from it. | -| `Rollup` | Rollup: `add_finer_aggregate_candidates` (`pass2/memo/rules/finer_aggregate.rs`), `add_rollup_candidates` and `add_window_rollup_candidates` (`pass2/memo/rules/rollup.rs`) | A coarse group computed from a finer group's values or states. | -| `SummaryResult` | `factor_summary_results` (`pass2/memo/rules/summary_results.rs`) | A summary readout factored out of a composite implementation, so that other readers can read it. | -| `SharedBuild` | Summary capability: `add_shared_summary_state_groups` (`pass2/memo/state_groups.rs`), Pass 2's last step | For each build made two or more times (by different candidates, or twice within one), one concrete summary-state group with one `SharedBuild` candidate; and for each candidate making such builds, one `SharedBuild` copy that reads the groups instead of building its own states. | - -The state-configuration rule (`pass2/memo/rules/shared_configuration_readers.rs`) runs just -before the shared builds are found. It rewrites readers to each shared size -and to the newest tumbling windows of each tumbling-window chain layout they may read, so that -equal builds appear, and a rewrite keeps the origin of the candidate it -rewrites. +| `TumblingWindows` | Chain groups: `add_chain_groups` (`stage1-pass2-reuse-candidates/src/rules/chains.rs`) | A chain group `SummaryTumblingWindowChain[w](X)` per width the reaching queries' evaluation grids allow, one build per surviving θ, the `TimeShift`/`TimeRange` groups over it, and the `SummaryMerge` over them in the state group. A chain's edge into a group whose evaluation grid its width does not divide fails with `ChainWidthDoesNotDivideEvaluationGrid`, so the merge over it is pruned: a 15 s chain never serves a state also read every 10 s. | +| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`stage1-pass2-reuse-candidates/src/rules/topk.rs`) | For each exact top-k, `Aggregate[TopK k]` over the ranked rows of the top-k group of the same input and partition with the largest `k`, partitioned by that group's leading key columns: the `k` highest of the `k_max` highest rows are the `k` highest. | +| `Rollup` | Rollup from the finest common group: `add_rollup_candidates` (`stage1-pass2-reuse-candidates/src/rules/rollup.rs`) | A coarse group computed from a finer group's values or states (through a compensating `Filter` on the finer group's keys), and the builds of a cluster's finest common group. | **Each stage over the memo.** Each stage keeps its decision, on groups instead of whole candidates: @@ -392,48 +382,29 @@ stands for 282 = 2 × (1 + 2 × 7 + 2 × 7 + 7 + 7 + 2 × 7 × 7) candidate DAGs: G6 exact or from accumulators, times G2 and G4 both exact, one exact and the other from KLL or DDSketch, both from one family, or one from each. -Steps 3 and 4, and their rows in the table after step 4, predate this Pass 1 -sizing; their counts are not recomputed here. - -**Step 3, Stage 1 Pass 2: window composition.** The groups stay the same 7. -G2 and G4 get 120 `TumblingWindows` candidates each (6 summary candidates × 20 tumbling window -widths, from 1 s to 60 s, that divide one minute), 127 candidates each in -all; the memo has 259 candidates. - -**Step 4, the other Pass 2 rules and the concrete summary-state groups** -(rules 2–9 of `generate_logical_asap_memo_candidates`, then -`add_shared_summary_state_groups`): -- 354 groups: the 7 above; 336 `SummaryTumblingWindowChain` groups (7 summary configurations - × 48 tumbling-window layouts, with tumbling-window widths from 20 ms to 60 s); 7 `SummaryAgg` - state groups (the 6 summaries above and a KLL with `k` 1156, - `failure_probability` 1e-7); 2 `SummaryAgg` groups for G6's sum and count - accumulators; and 2 `FinalizeExactAccumulator` groups (G6's sum and count - readouts). -- 1738 candidates. G2 and G4 have 691 each: 10 `Pass1` (the 7 above and - 3 copies of the KLL `k` 567 that the state-configuration - rule rewrote to the shared configurations with `k` 567, 1156 and 1184; the - `k` 1184 one composes the same DAG as the KLL `k` 1184), - 336 `TumblingWindows` and 345 `SharedBuild`. G6 has 3: `Pass1`, `SummaryResult` and - `SharedBuild`. -- Enumeration stops at the former `stage_pipeline`'s default limit of 1,000 candidate - DAGs (`--memo-enumeration-limit`), as it does after step 3. +**Step 3, Stage 1 Pass 2** (`generate_logical_asap_memo_candidates`). +The workload has no second aggregate of the same definition (no rollup), no +top-k (no top-k reuse) and no evaluation grid (no chain group), so the +groups and candidates stay 19 and 36. The query's target is an absolute +error, which the `+` carries to G4 and the division to neither of its +operands: the metric rule cuts every edge from a KLL readout (rank error) +or a DDSketch readout (relative error) into G2 and G4, so no readout gets a +required accuracy. Dead-candidate elimination removes those 4 candidates of +G2 and G4, the 4 readouts and the 14 builds (22 pruned); 14 candidates +survive, with 16 logical edges. The surviving choices are G6 exact or from +the accumulators, over exact quantiles: 2 candidate DAGs. | After | Groups | Candidates for memo groups | Candidate DAGs | |---|---|---|---| | Stage 0 | 7 | – (each group is its own node) | 1 | | Stage 1, Pass 1 | 19 | 36 | 282 | -| Stage 1, Pass 2 window composition | 7 | 259 | over 1,000 (enumeration stopped at its limit) | -| Stage 1, Pass 2 rules and concrete summary-state groups | 354 | 1738 | over 1,000 (enumeration stopped at its limit) | - -These numbers are the production planner's, read from the stage-pipeline -document `tools/dag-viewer/examples/memo-ratio-of-quantiles.json` (made by -the former `stage_pipeline` from `tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json`, -with accuracy guarantees enforced and the `aws-fargate-us-east-1` cost model). The Stage 1, Pass 1 row is -`memo_of_queries` then `pass1_memo` run on that workload with the -executor's summary families, at the current Pass 1 sizing. No test asserts -these numbers. The last -step is Pass 2's memo over the complete tumbling-window-width domain, which -gives 438 groups and 2158 candidates. +| Stage 1, Pass 2 | 19 | 36, 14 surviving dead-candidate elimination | 2 | + +These numbers are the production planner's: `memo_of_queries`, then +`pass1_memo` and `generate_logical_asap_memo_candidates`, run on +`tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json` with the +executor's summary families and accuracy guarantees enforced. No test +asserts them. #### Algorithm: memo to candidate DAGs @@ -477,7 +448,7 @@ that vector. | Reason | Effect on the count | In the example | |---|---|---| | Shared group | A group several chosen candidates read takes one candidate per choice, not one per reader. | G1 is read by G2 and G4 and chosen once. | -| Group needed only under some choices | A group joins the choice only when a chosen candidate reads it. | A `SummaryTumblingWindowChain` group is needed only when a `SharedBuild` candidate reading it is chosen; a choice of `Pass1` candidates needs none of the 347 added groups. | +| Group needed only under some choices | A group joins the choice only when a chosen candidate reads it. | The KLL state group G7 is needed only when a candidate reading G8 or G11 is chosen; a choice of exact candidates needs none of G7–G12. | | Fused candidate | The groups a fused candidate implements are computed inside it and never become pending. | None (the example has no fused computation). | In Pass 1 of the example the count is not a product either: G2 and G4 @@ -518,19 +489,20 @@ full tumbling-window-width domain, with no time limit. #### Memo code interface -The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (module -`memo`, crate-private: the crate root re-exports each entry point); the memo types Stage 1 hands Stage 2 are in +The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (files +`rules/`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, +`evaluation_grids.rs` and `exact_only_groups.rs`, crate-private: the crate +root's `generate_logical_asap_memo_candidates` runs them); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| -| Stage 0 | `memo_of_queries` → `Memo` (without candidates), `MemoGroup`, `MemoGroupId` | `stage0-memo/src/lib.rs` (the types: `asap_types::stages`); hash-consing is `Memo::find_or_add_group` in `types/src/stages/logical/memo/mod.rs`, its expression equality in `memo/hash_consing.rs` | +| Stage 0 | `memo_of_queries` → `Memo` (without candidates), `MemoGroup`, `MemoGroupId` | `stage0-memo/src/lib.rs` (the types: `asap_types::stages`); hash-consing is `Memo::find_or_add_group` in `types/src/stages/logical/memo/mod.rs`, its expression equality in `memo/hash_consing/mod.rs` | | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | -| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `stage1-pass2-reuse-candidates/src/lib.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | -| Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | +| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{rollup,topk,chains}.rs`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`, `exact_only_groups.rs`; the readout levels: `asap_accuracy::readout_levels`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | | Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -599,8 +571,7 @@ the memo path's entry points. |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (no longer in the code) | Logical roots → Stage 0's `Memo` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support → the same `Memo` with each group's `CandidateForMemoGroup`s (a summary with one build per configuration of its `parameter_list`) | `memo_of_queries`, `pass1_memo` | -| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | -| 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | +| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → `LogicalASAPMemoCandidates`: the same memo with the rules' groups and candidates added (rollup, top-k reuse, chain groups), each readout's required accuracy, the logical edges between the surviving candidates, the pruned candidates and, per query, the groups it may read only exactly | `generate_logical_asap_memo_candidates` (`asap_stage1_pass2_reuse_candidates`) | | 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -1226,8 +1197,14 @@ agree, and the interval and a declared phase are multiples of the window: `m[5m]` every 5 minutes is one tumbling 5-minute window maintained at ingestion time and read at each evaluation, as a one-tumbling-window chain would be, while `m[20s]` every 20 s at phase 5 s, or every 30 s, is not kept whole (Pass 2 offers 5 s or 10 s tumbling windows). A -reader whose phase is not declared is taken to evaluate at multiples of its -interval, as for a tumbling-window chain. +reader whose phase is not declared has no known evaluation grid +(`QueryRecurrence::evaluation_grid_ms` is `None`: Prometheus staggers rule groups +by a hash, so a fixed interval does not mean phase 0), and Pass 2 adds no +tumbling-window chain for a state it reads. Below a PromQL subquery the +grid is the subquery's: `[10m:30s]` evaluates `` at the +multiples of 30 s whatever the outer grid, and `[10m:]` at the +server's default interval, which the planner does not know, so a state +there gets no chain. #### Physical operator implementation diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 340e25183..0ab52dfbe 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -22,7 +22,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | `asap-stage0-memo` (`crates/stage0-memo`) | Stage 0 hash-consing: the memo groups of the workload's query roots (`memo_of_queries`) | | `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates`) | Stage 1, Pass 2: the reuse rules over memo groups and the logical edges | -| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, composition, budget split, target check), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | +| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, each operator's error propagation, the target comparison limit), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | | `asap-stage2-physical-optimization` (`crates/stage2-physical-optimization`) | Stage 2: materialization and physical operator implementation | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | | `asap-stage3-plan-selection` (`crates/stage3-plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | @@ -79,7 +79,8 @@ same pipeline behind the `OptimizationPass` trait. | Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-stage3-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | -| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `requirements.rs`, `error_propagation.rs`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | +| The accuracy rules: per-family guarantees, each operator's error propagation, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_propagation.rs`, `metric_rule.rs`, `readout_caps.rs`, `logical_edge.rs`) | +| The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | | What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `models.rs` (`PlanningModels`) and `pass.rs` (the trait); the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 8bbd9da5c..57a9154fb 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -69,8 +69,9 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- - A target with no `accuracy_metric` accepts a guarantee in any metric that meets its ε, δ and scope; Stage 3 reports the metric it accepted (`AcceptedAccuracy`). -- An approximate input split across several inputs gets its share through - `accuracy::budget::per_input_target`. +- A query that composes several readouts gives each readout a level of an + error and failure-probability grid, within the cap its path's consumers + (`accuracy::error_propagation`) leave it. - Facts come only from the declared `DataWorkload`; an undeclared fact is unknown, never zero. diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 43aa9641b..3565bad7e 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -19,9 +19,9 @@ aggregate intent and AccuracyTarget -> choose and commit concrete summary parameters -> derive the local guarantee from those committed parameters -> realize the child and obtain its finalized guarantee - -> propagate guarantees through the registered composition rule + -> propagate required errors through the operators' error contracts -> check the final guarantee against AccuracyTarget - -> keep legal candidate or record AccuracyError + -> keep legal candidate or record the pruning reason -> Stage 3 prices the valid candidates under the cost model (cost does not prove accuracy) ``` @@ -75,79 +75,96 @@ pub trait AccuracyEvidence { } ``` -`accuracy::budget::per_input_target` splits a target equally among the -approximate inputs of one composition (ε / n for an additive metric, -1 − (1 + ε)^(−1/n) for a relative one, δ / n). The relative share meets the -composed bound of one product or quotient of two inputs: a ratio of two -relative errors `a` and `b` is within `(a + b) / (1 − b)`, so ε = 0.05 gives -each side 0.0241. A nested quotient (`x / (y / z)`) or an exact numerator over a -lone approximate denominator composes a bound above ε and is rejected. Every candidate must then be resized, propagated, and -checked before being treated as satisfying the target. This is distinct from +Pass 2 gives each sketch readout the accuracy it is required to meet: a +query's target divided by the propagation factors of the consumers on the path +from the readout to the query root (`accuracy::error_propagation`, a linear +bound per operator and metric, with a validity cap on an operand where the +bound holds only below it, such as the denominator of a ratio). A query that +reads one readout gives it one level; a query that composes several readouts +(a ratio of two quantiles) gives every level of an error and failure-probability +grid not looser than the cap, and each logical edge is then checked against +the levels. A consumer with no contract for a metric fails the edge from a +readout in that metric (`RootPruneReason::MetricRule`), so an +approximate value is never treated as exact. Every candidate is +checked against the target before being treated as satisfying it. This is distinct from candidate visibility: a Stage 1 candidate whose output has no guarantee is still enumerated and appears in the stage document, but Stage 3 rejects it for any approximate target. Presence and cost are not accuracy certification. See the [workflow design](../design_docs/architecture/input-output-workflow.md#planning-evidence-inputs) for this boundary. -## Composition contracts +## Error contracts -### Exact values - -An exact value contributes zero error and zero failure probability: - -```text -B_exact = 0 -delta_exact = 0 -``` - -An exact input does not exempt a local approximate sketch from target checking. - -### Additive and Lipschitz composition - -Compatible absolute bounds compose without an independence assumption: +Only a sketch readout (`SummaryEstimate` over a sketch state) introduces +error. Every other operator passes its inputs' errors on through its error +contract, `asap_accuracy::error_propagation(memo, consumer, metric)` +(`accuracy/src/error_propagation.rs`): a linear bound on the output error +from the input errors in one metric, ```text -B_total <= B_input + B_local -delta_total <= delta_input + delta_local +error_output <= sum_i factor_i * error_i while error_i <= validity_cap_i ``` -For a registered `L`-Lipschitz transformation: - -```text -B_output <= L * B_input + B_local -delta_output <= delta_input + delta_local -``` - -Failure probabilities use a union bound. - -### Relative composition - -For a registered multiplicative rule whose values are known to be -non-negative: - -```text -epsilon_total = - epsilon_input - + epsilon_local - + epsilon_input * epsilon_local -``` - -Reject the rule when sign information is missing or values may cross zero. - -### Exact aggregation over approximate values - -For an exact sum, sum the input bounds and union-bound input failures. If the -number of folded rows is required but unknown, keep the result symbolic. - -For exact minimum or maximum, use the maximum input value-error bound and -union-bound failures. This does not prove the identity of a winning key. - -### Unsupported composition - -Accept approximate-over-approximate composition only when a rule exists for -the operator and metric. Cross-metric and unregistered same-metric composition -must return `UnsupportedComposition`; never treat an approximate child as -exact. +or `None` when the operator has no contract for that metric. A contract +keeps the metric: an input's error and the output's are in the same one. + +| Operator | Metric | Factor per input | Validity cap | +| --- | --- | --- | --- | +| `Sort`, `TimeRange`, `TimeShift`, `Concat`, a readout, a finalize, `SummaryMerge` | any | 1 | none | +| `Filter` whose predicate reads only columns copied from a source (labels, keys) | any | 1 | none | +| `Project` whose items reading a computed value (not a copied label, the time or the series identity) are copies | any | 1 | none | +| the same, with items computing `abs`, `clamp`, `clamp_min`, `clamp_max` (or PromQL's `promql_abs`, …) or a negation of a column | absolute | 1 | none | +| the same, with items multiplying or dividing a column by a nonzero literal | relative | 1 | none | +| `Aggregate` (or exact accumulator build) by keys copied from a source, without measure filters or `HAVING`: `count` | any | 0 | none | +| the same: `min`, `max` | absolute, relative | 1 | none | +| the same: `avg` | absolute | 1 | none | +| `+`, `-` | absolute | 1 | none | +| `/` | relative | `1 / (1 - b)` on both | `b` on the denominator | +| `*` | relative | `1 + b` on the left, 1 on the right | `b` on the right operand | + +`b` is `RELATIVE_OPERAND_VALIDITY_CAP` (0.1, the weakest error of every +parameter list). Every other operator has no contract: a filter or grouping +on a computed value, a ranking, a `sum`, a comparison, any other function. + +The contracts are used in three places: + +- **The metric rule** (`asap_accuracy::accepted_metrics`, + `accuracy/src/metric_rule.rs`): a query root accepts its target's metric + (every metric when the target names none); an input of a candidate + accepts the metrics its output accepts that the candidate's contract + carries. An edge from a readout whose metric its consumer does not + accept fails (`RootPruneReason::MetricRule`), so an approximate value + never reaches an operator with no contract for its error. Where no query + with a target reaches a group, no metric rule applies. The rule is a + union over the queries reaching a group; per query, Pass 2 lists the + groups reached through a consumer without a contract for the query's + metric (`exact_only_groups_of_query`, `exact_only_groups.rs` in Pass 2), + which Stage 3 must let that query read only exactly. +- **Readout requirements** (`asap_accuracy::readout_levels`, + `accuracy/src/readout_caps.rs`): each query's target is passed from its + root down to the readouts, divided by each consumer's factor and capped + by its validity cap, keeping per group the largest cap of each query + and metric. A query that reads one readout gives it one level, its cap + at the target's failure probability per value (a whole-result δ split + over the larger of the values one evaluation returns and the readout + group's values per evaluation, since a `max` over many readouts fails + when any of them does); a count passes no cap down; a query that combines several + readouts (a ratio of two quantiles) gives every level of the grid + `MAX_ERROR_LEVELS` × a failure-probability grid (powers of ten from 1e-2 + down to 1e-5, continued to the first at or below a tenth of the per-value + δ) that is not looser than the cap and the per-value δ. Pass 2 writes each level on a readout + candidate of its own. +- **$\alpha^{\log}$** (`asap_accuracy::producer_satisfies_logical_requirements`, + `accuracy/src/logical_edge.rs`): an edge into a readout holds when the + producer's θ has a guarantee (measured, else `theoretical_guarantee`) + in the required metric within the required error at the required + failure probability, and a merged state keeps the guarantee of a build. + Under a query with a target, a readout with no guarantee, or one whose + failure probability is unknown (HLL's), fails with `NoGuarantee`. + +A value that is exact has no error and needs no contract. Nothing composes +failure probabilities across readouts beyond the per-value split and the +grid above, and nothing adds a local error at a non-readout operator. ## Built-in sketch contracts @@ -199,18 +216,11 @@ epsilon_cardinality = 1.04 / sqrt(2^p) delta = unknown ``` -Sizing inverts the magnitude. HLL's current parameter model contains precision -only; it does not encode a confidence-level budget. Generic HLL may therefore -satisfy a target with only `error` set, but not one that also sets -`failure_probability`. - -Classic HLL (its linear-counting branch) has an estimator-specific confidence -bound (`accuracy/guarantee/hll.rs`). Its precision is sized from (ε, δ), and -its failure probability stays symbolic, -`classic_hll_failure_probability[p=P,eps=ε](max_distinct[source])`, until -plan selection binds it with the source's -`DataWorkload.sources[source].max_distinct_per_evaluation`. A missing bound -leaves the target unmet. +HLL's parameter model contains precision only; it does not encode a +confidence-level budget. With its failure probability unknown, a distinct +count read from HLL serves no query with an accuracy target (the edge fails +with `NoGuarantee`); KMV is the distinct-count summary with a certified +bound. ### KMV and Theta @@ -261,27 +271,17 @@ with probability at most `1/3`. Zero or even depth has no modeled guarantee. ASAPPlanner contains the guarantee algebra and parameter-derived contracts. Planner is not a query executor. Data- or runtime-dependent evidence enters through `AccuracyEvidence` (measured guarantees) or the data workload's -declared facts, is exposed to propagation as typed `PropagationStats`, and -is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. +declared facts, and is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. ### TopK membership -`PropagationStats` supplies: - -- the lower confidence bound of the kth selected item; -- the greatest upper confidence bound among excluded items; and -- the union-bound failure probability of all certificate intervals. - -The intervals must already include underlying sketch error. Certify exact -membership only when: - -```text -selected_kth_lower_bound > excluded_max_upper_bound -``` - -The result uses `TopKMembership`, zero membership error, and the supplied -failure probability. Missing, non-finite, invalid, equal, or overlapping -evidence rejects the candidate. +Planner has no membership certificate for a top-k. A top-k readout from a +heap variant (`CmsWithHeap`, `CountSketchWithHeap`) is admitted when its +heap holds at least `k` entries (else the edge fails with +`RootPruneReason::HeapSmallerThanTopK`) and the counts it reports meet the +requirement: the Count-Min L1 or Count Sketch L2 absolute stream bound of +the sketch's width and depth. Whether the right `k` keys are selected is not +checked on its own. ### Hydra shared grid @@ -296,6 +296,8 @@ When observations are unavailable, preserve `hydra_shared_grid_collision_bound` and `hydra_shared_grid_failure_probability` as symbolic leaves and reject an accuracy-targeted candidate. Never copy the inner guarantee onto Hydra alone. +The code does not implement this yet: a Hydra state currently has its +per-subpopulation state's guarantee, without the shared-grid term. ## Adding or changing an accuracy target @@ -308,23 +310,26 @@ optional `accuracy_metric` and an `accuracy_scope`; built with `exact()` or 2. Define the parameters and evidence needed to prove satisfaction. 3. Add the satisfaction rule, including invalid, unknown, and unsupported behavior. -4. Add allocation behavior where the target can be divided across layers; do - not invent allocation for non-divisible semantics. +4. Add how the target passes through the error contracts down to the + readouts; do not invent a split for a target that does not divide. 5. Add serialization, DAG explanation, and compatibility behavior. -6. Add local and composition contracts for summaries that claim the target. +6. Add the readout guarantee for summaries that claim the target, and an + error contract for each operator its error must pass through. 7. Keep candidates fail-closed until the end-to-end guarantee path is implemented and tested. Do not coerce membership, distributional, or another non-epsilon requirement into an existing variant merely to reuse its API. -## Adding a summary or composition +## Adding a summary or an error contract -For a new summary, register its error metric and a local contract tied to the -same parameters that sizing commits. Record estimator/version premises in -provenance. For a new composition, specify compatible input and output metrics, -the bound rule, failure composition, required statistics, and unsupported -cases. Missing evidence must stay symbolic or produce a structured rejection. +For a new summary, give its readouts a guarantee per listed θ at a failure +probability (`guarantee/`) in one metric, tied to the parameters its list +commits. Record estimator/version premises in provenance. For a new error +contract, add the operator's row to `error_propagation`: the metric it +keeps, the factor per input and any validity cap. An operator without a +row has no contract, so an approximate input never reaches it. Missing +evidence must stay symbolic or produce a structured rejection. Update DAG export whenever new guarantee or rejection data is introduced. @@ -332,8 +337,9 @@ Update DAG export whenever new guarantee or rejection data is introduced. Tests must cover: -- exact, additive, relative, Lipschitz, sum, and extremum propagation; -- incompatible metrics and unsupported composition; +- each operator's error contract per metric, and operators without one; +- the metric rule and the readout levels a target gives through the + contracts; - malformed and unknown symbolic values; - target checking after parameter clamps; - CountSketch L2 sizing distinct from CMS L1 sizing; diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 2cb7da985..1b303529a 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -78,7 +78,7 @@ contains the candidate and that Stage 3 prices it. ## 3. Add a Stage 2 physical properties option 1. Add the `Lifetime` variant or the valid (timing, lifetime) pair - (`crates/types/src/stages/physical/properties.rs`, `PhysicalProperties::new` and + (`crates/types/src/stages/physical/properties/mod.rs`, `PhysicalProperties::new` and `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/stage2-physical-optimization/src/lib.rs` (the diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 070c018ce..8a7b4f3b0 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -32,7 +32,6 @@ pub struct OperatorNode { pub operator: Operator, // NonASAP(NonASAPOp) | ASAP(ASAPOp) pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction - pub guarantee: Option, // None until accuracy assessment establishes one } ``` @@ -44,7 +43,6 @@ pub struct OperatorNode { - `schema` is derived by `OperatorNode::new(operator)`; it fails when the schema cannot be derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. -- `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. - A node has no physical properties (timing, lifetime): a logical candidate has only its logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` @@ -148,7 +146,7 @@ yet — and calls `asap_frontend_common::resolve_root`, which does three things The result is `Rc`. `lower_sql` / `lower_sql_dialect` and `lower_metricsql` return it; `lower_promql_query_workload` returns a `QueryRoot` per query. -Workload search then runs structural CSE (Stage 0's `memo_of_queries` in `crates/stage0-memo/src/lib.rs`, hash-consing each operator into a `Memo` group through `Memo::find_or_add_group` and `crates/types/src/stages/logical/memo/hash_consing.rs`) once +Workload search then runs structural CSE (Stage 0's `memo_of_queries` in `crates/stage0-memo/src/lib.rs`, hash-consing each operator into a `Memo` group through `Memo::find_or_add_group` and `crates/types/src/stages/logical/memo/hash_consing/mod.rs`) once across every root: bottom-up hash-consing where the structural hash is only a filter and the typed `PartialEq` decides sharing, following scalar references like any other input, and gated by `NonASAPOp::is_shareable_when_equal` for non-ASAP producers. diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 1f6564ca2..b59e5160c 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -49,7 +49,7 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | | Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | -| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `memo_of_queries` (`asap_stage0_memo`, `crates/stage0-memo/src/lib.rs`), hash-consing through `Memo::find_or_add_group` (`crates/types/src/stages/logical/memo/hash_consing.rs`) | +| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `memo_of_queries` (`asap_stage0_memo`, `crates/stage0-memo/src/lib.rs`), hash-consing through `Memo::find_or_add_group` (`crates/types/src/stages/logical/memo/hash_consing/mod.rs`) | | `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::plan::physical_asap_dag::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | | Exported schema JSON `columns` | `fields` | diff --git a/docs/user_guide_docs/asap-planner.md b/docs/user_guide_docs/asap-planner.md index 3b29c71f9..f8c293b0f 100644 --- a/docs/user_guide_docs/asap-planner.md +++ b/docs/user_guide_docs/asap-planner.md @@ -62,7 +62,7 @@ under the stage that failed. | --- | --- | | `stage0` | The frontends' query DAGs and their memo groups. | | `stage1_pass1` | Stage 1 Pass 1: each computation's candidates, and the candidates pruned with their reason. | -| `stage1_pass2` | Stage 1 Pass 2: the memo after local window composition (a candidate for every legal tumbling-window width), then the memo after the sharing rules, which Stage 2 implements. | +| `stage1_pass2` | Stage 1 Pass 2: the memo after the reuse rules and chain groups (a chain group for every legal tumbling-window width), with the logical edges and the pruned candidates, which Stage 2 implements. | | `stage2` | Stage 2: the physical candidates and the ones pruned. | | `stage3` | Stage 3: the selected plan, its cost per node, each query's accepted accuracy and the search status. |